http://git-wip-us.apache.org/repos/asf/incubator-predictionio-site/blob/4e16ed29/customize/troubleshooting/index.html ---------------------------------------------------------------------- diff --git a/customize/troubleshooting/index.html b/customize/troubleshooting/index.html new file mode 100644 index 0000000..9daedeb --- /dev/null +++ b/customize/troubleshooting/index.html @@ -0,0 +1,85 @@ +<!DOCTYPE html><html><head><title>Engine Development - Troubleshoot</title><meta charset="utf-8"/><meta content="IE=edge,chrome=1" http-equiv="X-UA-Compatible"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta class="swiftype" name="title" data-type="string" content="Engine Development - Troubleshoot"/><link rel="canonical" href="https://predictionio.incubator.apache.org/customize/troubleshooting/"/><link href="/images/favicon/normal-b330020a.png" rel="shortcut icon"/><link href="/images/favicon/apple-c0febcf2.png" rel="apple-touch-icon"/><link href="//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800" rel="stylesheet"/><link href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css" rel="stylesheet"/><link href="/stylesheets/application-3a3867f7.css" rel="stylesheet" type="text/css"/><script src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.2/html5shiv.min.js"></script><scr ipt src="//cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script><script src="//use.typekit.net/pqo0itb.js"></script><script>try{Typekit.load({ async: true });}catch(e){}</script></head><body><div id="global"><header><div class="container" id="header-wrapper"><div class="row"><div class="col-sm-12"><div id="logo-wrapper"><span id="drawer-toggle"></span><a href="#"></a><a href="http://predictionio.incubator.apache.org/";><img alt="PredictionIO" id="logo" src="/images/logos/logo-ee2b9bb3.png"/></a></div><div id="menu-wrapper"><div id="pill-wrapper"><a class="pill left" href="/gallery/template-gallery">TEMPLATES</a> <a class="pill right" href="//github.com/apache/incubator-predictionio/">OPEN SOURCE</a></div></div><img class="mobile-search-bar-toggler hidden-md hidden-lg" src="/images/icons/search-glass-704bd4ff.png"/></div></div></div></header><div id="search-bar-row-wrapper"><div class="container-fluid" id="search-bar-row"><div class="row"><div class="col-md -9 col-sm-11 col-xs-11"><div class="hidden-md hidden-lg" id="mobile-page-heading-wrapper"><p>PredictionIO Docs</p><h4>Troubleshooting Engine Development</h4></div><h4 class="hidden-sm hidden-xs">PredictionIO Docs</h4></div><div class="col-md-3 col-sm-1 col-xs-1 hidden-md hidden-lg"><img id="left-menu-indicator" src="/images/icons/down-arrow-dfe9f7fe.png"/></div><div class="col-md-3 col-sm-12 col-xs-12 swiftype-wrapper"><div class="swiftype"><form class="search-form"><img class="search-box-toggler hidden-xs hidden-sm" src="/images/icons/search-glass-704bd4ff.png"/><div class="search-box"><img src="/images/icons/search-glass-704bd4ff.png"/><input type="text" id="st-search-input" class="st-search-input" placeholder="Search Doc..."/></div><img class="swiftype-row-hider hidden-md hidden-lg" src="/images/icons/drawer-toggle-active-fcbef12a.png"/></form></div></div><div class="mobile-left-menu-toggler hidden-md hidden-lg"></div></div></div></div><div id="page" class="container-fluid"><div class="row"><div id="left-menu-wrapper" class="col-md-3"><nav id="nav-main"><ul><li class="level-1"><a class="expandible" href="/"><span>Apache PredictionIO (incubating) Documentation</span></a><ul><li class="level-2"><a class="final" href="/"><span>Welcome to Apache PredictionIO (incubating)</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Getting Started</span></a><ul><li class="level-2"><a class="final" href="/start/"><span>A Quick Intro</span></a></li><li class="level-2"><a class="final" href="/install/"><span>Installing Apache PredictionIO (incubating)</span></a></li><li class="level-2"><a class="final" href="/start/download/"><span>Downloading an Engine Template</span></a></li><li class="level-2"><a class="final" href="/start/deploy/"><span>Deploying Your First Engine</span></a></li><li class="level-2"><a class="final" href="/start/customize/"><span>Customizing the Engine</span></a></li></ul></li><li class="level-1"><a class="expandible" href=" #"><span>Integrating with Your App</span></a><ul><li class="level-2"><a class="final" href="/appintegration/"><span>App Integration Overview</span></a></li><li class="level-2"><a class="expandible" href="/sdk/"><span>List of SDKs</span></a><ul><li class="level-3"><a class="final" href="/sdk/java/"><span>Java & Android SDK</span></a></li><li class="level-3"><a class="final" href="/sdk/php/"><span>PHP SDK</span></a></li><li class="level-3"><a class="final" href="/sdk/python/"><span>Python SDK</span></a></li><li class="level-3"><a class="final" href="/sdk/ruby/"><span>Ruby SDK</span></a></li><li class="level-3"><a class="final" href="/sdk/community/"><span>Community Powered SDKs</span></a></li></ul></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Deploying an Engine</span></a><ul><li class="level-2"><a class="final" href="/deploy/"><span>Deploying as a Web Service</span></a></li><li class="level-2"><a class="final" href="/batchpredict/"><span>Batch Predictions</s pan></a></li><li class="level-2"><a class="final" href="/deploy/monitoring/"><span>Monitoring Engine</span></a></li><li class="level-2"><a class="final" href="/deploy/engineparams/"><span>Setting Engine Parameters</span></a></li><li class="level-2"><a class="final" href="/deploy/enginevariants/"><span>Deploying Multiple Engine Variants</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Customizing an Engine</span></a><ul><li class="level-2"><a class="final" href="/customize/"><span>Learning DASE</span></a></li><li class="level-2"><a class="final" href="/customize/dase/"><span>Implement DASE</span></a></li><li class="level-2"><a class="final active" href="/customize/troubleshooting/"><span>Troubleshooting Engine Development</span></a></li><li class="level-2"><a class="final" href="/api/current/#package"><span>Engine Scala APIs</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Collecting and Analyzing Data</span></a><ul><l i class="level-2"><a class="final" href="/datacollection/"><span>Event Server Overview</span></a></li><li class="level-2"><a class="final" href="/datacollection/eventapi/"><span>Collecting Data with REST/SDKs</span></a></li><li class="level-2"><a class="final" href="/datacollection/eventmodel/"><span>Events Modeling</span></a></li><li class="level-2"><a class="final" href="/datacollection/webhooks/"><span>Unifying Multichannel Data with Webhooks</span></a></li><li class="level-2"><a class="final" href="/datacollection/channel/"><span>Channel</span></a></li><li class="level-2"><a class="final" href="/datacollection/batchimport/"><span>Importing Data in Batch</span></a></li><li class="level-2"><a class="final" href="/datacollection/analytics/"><span>Using Analytics Tools</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Choosing an Algorithm(s)</span></a><ul><li class="level-2"><a class="final" href="/algorithm/"><span>Built-in Algorithm Libraries</span ></a></li><li class="level-2"><a class="final" >href="/algorithm/switch/"><span>Switching to Another >Algorithm</span></a></li><li class="level-2"><a class="final" >href="/algorithm/multiple/"><span>Combining Multiple >Algorithms</span></a></li><li class="level-2"><a class="final" >href="/algorithm/custom/"><span>Adding Your Own >Algorithms</span></a></li></ul></li><li class="level-1"><a class="expandible" >href="#"><span>ML Tuning and Evaluation</span></a><ul><li class="level-2"><a >class="final" href="/evaluation/"><span>Overview</span></a></li><li >class="level-2"><a class="final" >href="/evaluation/paramtuning/"><span>Hyperparameter >Tuning</span></a></li><li class="level-2"><a class="final" >href="/evaluation/evaluationdashboard/"><span>Evaluation >Dashboard</span></a></li><li class="level-2"><a class="final" >href="/evaluation/metricchoose/"><span>Choosing Evaluation >Metrics</span></a></li><li class="level-2"><a class="final" >href="/evaluation/metricbuild/"><span>Building Evaluation Metrics </span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>System Architecture</span></a><ul><li class="level-2"><a class="final" href="/system/"><span>Architecture Overview</span></a></li><li class="level-2"><a class="final" href="/system/anotherdatastore/"><span>Using Another Data Store</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>PredictionIO Official Templates</span></a><ul><li class="level-2"><a class="final" href="/templates/"><span>Intro</span></a></li><li class="level-2"><a class="expandible" href="#"><span>Recommendation</span></a><ul><li class="level-3"><a class="final" href="/templates/recommendation/quickstart/"><span>Quick Start</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/dase/"><span>DASE</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/evaluation/"><span>Evaluation Explained</span></a></li><li class="level-3"><a class="final" href="/t emplates/recommendation/how-to/"><span>How-To</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/reading-custom-events/"><span>Read Custom Events</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/customize-data-prep/"><span>Customize Data Preparator</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/customize-serving/"><span>Customize Serving</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/training-with-implicit-preference/"><span>Train with Implicit Preference</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/blacklist-items/"><span>Filter Recommended Items by Blacklist in Query</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/batch-evaluator/"><span>Batch Persistable Evaluator</span></a></li></ul></li><li class="level-2"><a class="expandible" href="#"><span>E-Commerce Recommendat ion</span></a><ul><li class="level-3"><a class="final" href="/templates/ecommercerecommendation/quickstart/"><span>Quick Start</span></a></li><li class="level-3"><a class="final" href="/templates/ecommercerecommendation/dase/"><span>DASE</span></a></li><li class="level-3"><a class="final" href="/templates/ecommercerecommendation/how-to/"><span>How-To</span></a></li><li class="level-3"><a class="final" href="/templates/ecommercerecommendation/train-with-rate-event/"><span>Train with Rate Event</span></a></li><li class="level-3"><a class="final" href="/templates/ecommercerecommendation/adjust-score/"><span>Adjust Score</span></a></li></ul></li><li class="level-2"><a class="expandible" href="#"><span>Similar Product</span></a><ul><li class="level-3"><a class="final" href="/templates/similarproduct/quickstart/"><span>Quick Start</span></a></li><li class="level-3"><a class="final" href="/templates/similarproduct/dase/"><span>DASE</span></a></li><li class="level-3"><a class="final" href=" /templates/similarproduct/how-to/"><span>How-To</span></a></li><li class="level-3"><a class="final" href="/templates/similarproduct/multi-events-multi-algos/"><span>Multiple Events and Multiple Algorithms</span></a></li><li class="level-3"><a class="final" href="/templates/similarproduct/return-item-properties/"><span>Returns Item Properties</span></a></li><li class="level-3"><a class="final" href="/templates/similarproduct/train-with-rate-event/"><span>Train with Rate Event</span></a></li><li class="level-3"><a class="final" href="/templates/similarproduct/rid-user-set-event/"><span>Get Rid of Events for Users</span></a></li><li class="level-3"><a class="final" href="/templates/similarproduct/recommended-user/"><span>Recommend Users</span></a></li></ul></li><li class="level-2"><a class="expandible" href="#"><span>Classification</span></a><ul><li class="level-3"><a class="final" href="/templates/classification/quickstart/"><span>Quick Start</span></a></li><li class="level-3"><a clas s="final" href="/templates/classification/dase/"><span>DASE</span></a></li><li class="level-3"><a class="final" href="/templates/classification/how-to/"><span>How-To</span></a></li><li class="level-3"><a class="final" href="/templates/classification/add-algorithm/"><span>Use Alternative Algorithm</span></a></li><li class="level-3"><a class="final" href="/templates/classification/reading-custom-properties/"><span>Read Custom Properties</span></a></li></ul></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Engine Template Gallery</span></a><ul><li class="level-2"><a class="final" href="/gallery/template-gallery/"><span>Browse</span></a></li><li class="level-2"><a class="final" href="/community/submit-template/"><span>Submit your Engine as a Template</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Demo Tutorials</span></a><ul><li class="level-2"><a class="final" href="/demo/tapster/"><span>Comics Recommendation Demo</span></a></li> <li class="level-2"><a class="final" href="/demo/community/"><span>Community Contributed Demo</span></a></li><li class="level-2"><a class="final" href="/demo/textclassification/"><span>Text Classification Engine Tutorial</span></a></li></ul></li><li class="level-1"><a class="expandible" href="/community/"><span>Getting Involved</span></a><ul><li class="level-2"><a class="final" href="/community/contribute-code/"><span>Contribute Code</span></a></li><li class="level-2"><a class="final" href="/community/contribute-documentation/"><span>Contribute Documentation</span></a></li><li class="level-2"><a class="final" href="/community/contribute-sdk/"><span>Contribute a SDK</span></a></li><li class="level-2"><a class="final" href="/community/contribute-webhook/"><span>Contribute a Webhook</span></a></li><li class="level-2"><a class="final" href="/community/projects/"><span>Community Projects</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Getting Help</span> </a><ul><li class="level-2"><a class="final" href="/resources/faq/"><span>FAQs</span></a></li><li class="level-2"><a class="final" href="/support/"><span>Support</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Resources</span></a><ul><li class="level-2"><a class="final" href="/cli/"><span>Command-line Interface</span></a></li><li class="level-2"><a class="final" href="/resources/release/"><span>Release Cadence</span></a></li><li class="level-2"><a class="final" href="/resources/intellij/"><span>Developing Engines with IntelliJ IDEA</span></a></li><li class="level-2"><a class="final" href="/resources/upgrade/"><span>Upgrade Instructions</span></a></li><li class="level-2"><a class="final" href="/resources/glossary/"><span>Glossary</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Apache Software Foundation</span></a><ul><li class="level-2"><a class="final" href="https://www.apache.org/";><span>Apache Homepage</span></a>< /li><li class="level-2"><a class="final" href="https://www.apache.org/licenses/";><span>License</span></a></li><li class="level-2"><a class="final" href="https://www.apache.org/foundation/sponsorship.html";><span>Sponsorship</span></a></li><li class="level-2"><a class="final" href="https://www.apache.org/foundation/thanks.html";><span>Thanks</span></a></li><li class="level-2"><a class="final" href="https://www.apache.org/security/";><span>Security</span></a></li></ul></li></ul></nav></div><div class="col-md-9 col-sm-12"><div class="content-header hidden-md hidden-lg"><div id="breadcrumbs" class="hidden-sm hidden xs"><ul><li><a href="#">Customizing an Engine</a><span class="spacer">></span></li><li><span class="last">Troubleshooting Engine Development</span></li></ul></div><div id="page-title"><h1>Engine Development - Troubleshoot</h1></div></div><div id="table-of-content-wrapper"><h5>On this page</h5><aside id="table-of-contents"><ul> <li> <a href="#stop-training-between-stages">Stop Training between Stages</a> </li> <li> <a href="#sanity-check">Sanity Check</a> </li> <li> <a href="#engine-status-page">Engine Status Page</a> </li> <li> <a href="#pio-shell">pio-shell</a> </li> </ul> </aside><hr/><a id="edit-page-link" href="https://github.com/apache/incubator-predictionio/tree/livedoc/docs/manual/source/customize/troubleshooting.html.md";><img src="/images/icons/edit-pencil-d6c1bb3d.png"/>Edit this page</a></div><div class="content-header hidden-sm hidden-xs"><div id="breadcrumbs" class="hidden-sm hidden xs"><ul><li><a href="#">Customizing an Engine</a><span class="spacer">></span></li><li><span class="last">Troubleshooting Engine Development</span></li></ul></div><div id="page-title"><h1>Engine Development - Troubleshoot</h1></div></div><div class="content"> <p>Apache PredictionIO (incubating) provides the following features to help you debug engines during development cycle.</p><h2 id='stop-training-between-stages' class='header-anchors'>Stop Training betwee n Stages</h2><p>By default <code>pio train</code> runs through the whole training process including <a href="/templates/recommendation/dase/">DataSource, Preparator and Algorithm</a>. To speed up the development and debug cycle, you can stop the process after each stage to verify it has completed correctly.</p><p>If you have modified DataSource and want to confirm the TrainingData is generated as expected, you can run <code>pio train</code> with <code>--stop-after-read</code> option:</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1</pre></td><td class="code"><pre>pio train --stop-after-read +</pre></td></tr></tbody></table> </div> <p>This would stop the training process after the TrainingData is generated.</p><p>For example, if you are running <a href="/templates/recommendation/quickstart/">Recommendation Template</a>, you should see the the training process stops after the TrainingData is printed.</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4</pre></td><td class="code"><pre><span class="o">[</span>INFO] <span class="o">[</span>CoreWorkflow<span class="nv">$]</span> TrainingData: +<span class="o">[</span>INFO] <span class="o">[</span>CoreWorkflow<span class="nv">$]</span> ratings: <span class="o">[</span>1501] <span class="o">(</span>List<span class="o">(</span>Rating<span class="o">(</span>3,0,4.0<span class="o">)</span>, Rating<span class="o">(</span>3,1,4.0<span class="o">))</span>...<span class="o">)</span> +... +<span class="o">[</span>INFO] <span class="o">[</span>CoreWorkflow<span class="nv">$]</span> Training interrupted by org.apache.predictionio.workflow.StopAfterReadInterruption. +</pre></td></tr></tbody></table> </div> <p>Similarly, you can stop the training after the Preparator phase by using --stop-after-prepare option and it would stop after PreparedData is generated:</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1</pre></td><td class="code"><pre>pio train --stop-after-prepare +</pre></td></tr></tbody></table> </div> <h2 id='sanity-check' class='header-anchors'>Sanity Check</h2><p>You can extend a trait <code>SanityCheck</code> and implement the method <code>sanityCheck()</code> with your error checking code. The <code>sanityCheck()</code> is called when the data is generated. This can be applied to <code>TrainingData</code>, <code>PreparedData</code> and the <code>Model</code> classes, which are outputs of DataSource's <code>readTraining()</code>, Preparator's <code>prepare()</code> and Algorithm's <code>train()</code> methods, respectively.</p><p>For example, one frequent error with the Recommendation Template is that the TrainingData is empty because the DataSource is not reading data correctly. You can add the check of empty data inside the <code>sanityCheck()</code> function. You can easily add other checking logic into the <code>sanityCheck()</code> function based on your own needs. Also, If you implement <code>toString()</code> method in your TrainingData. You can call <code>toString()</code> inside <code>sanityCheck()</code> to print out some data for visual checking.</p><p>For example, to print TrainingData to console and check if the <code>ratings</code> is empty, you can do the following:</p><div class="highlight scala"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16</pre></td><td class="code"><pre><span class="k">import</span> <span class="nn">org.apache.predictionio.controller.SanityCheck</span> <span class="c1">// ADDED +</span> +<span class="k">class</span> <span class="nc">TrainingData</span><span class="o">(</span> + <span class="k">val</span> <span class="n">ratings</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[</span><span class="kt">Rating</span><span class="o">]</span> +<span class="o">)</span> <span class="k">extends</span> <span class="nc">Serializable</span> <span class="k">with</span> <span class="nc">SanityCheck</span> <span class="o">{</span> <span class="c1">// EXTEND SanityCheck +</span> <span class="k">override</span> <span class="k">def</span> <span class="n">toString</span> <span class="k">=</span> <span class="o">{</span> + <span class="n">s</span><span class="s">"ratings: [${ratings.count()}] (${ratings.take(2).toList}...)"</span> + <span class="o">}</span> + + <span class="c1">// IMPLEMENT sanityCheck() +</span> <span class="k">override</span> <span class="k">def</span> <span class="n">sanityCheck</span><span class="o">()</span><span class="k">:</span> <span class="kt">Unit</span> <span class="o">=</span> <span class="o">{</span> + <span class="n">println</span><span class="o">(</span><span class="n">toString</span><span class="o">())</span> + <span class="c1">// add your other checking here +</span> <span class="n">require</span><span class="o">(!</span><span class="n">ratings</span><span class="o">.</span><span class="n">take</span><span class="o">(</span><span class="mi">1</span><span class="o">).</span><span class="n">isEmpty</span><span class="o">,</span> <span class="n">s</span><span class="s">"ratings cannot be empty!"</span><span class="o">)</span> + <span class="o">}</span> +<span class="o">}</span> +</pre></td></tr></tbody></table> </div> <p>You may also use together with --stop-after-read flag to debug the DataSource:</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2</pre></td><td class="code"><pre>pio build +pio train --stop-after-read +</pre></td></tr></tbody></table> </div> <p>If your data is empty, you should see the following error thrown by the <code>sanityCheck()</code> function:</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4 +5 +6 +7 +8 +9</pre></td><td class="code"><pre><span class="o">[</span>INFO] <span class="o">[</span>CoreWorkflow<span class="nv">$]</span> Performing data sanity check on training data. +<span class="o">[</span>INFO] <span class="o">[</span>CoreWorkflow<span class="nv">$]</span> org.template.recommendation.TrainingData supports data sanity check. Performing check. +Exception <span class="k">in </span>thread <span class="s2">"main"</span> java.lang.IllegalArgumentException: requirement failed: ratings cannot be empty! + at scala.Predef<span class="nv">$.</span>require<span class="o">(</span>Predef.scala:233<span class="o">)</span> + at org.template.recommendation.TrainingData.sanityCheck<span class="o">(</span>DataSource.scala:73<span class="o">)</span> + at org.apache.predictionio.workflow.CoreWorkflow<span class="nv">$$</span>anonfun<span class="nv">$runTypelessContext$7</span>.apply<span class="o">(</span>Workflow.scala:474<span class="o">)</span> + at org.apache.predictionio.workflow.CoreWorkflow<span class="nv">$$</span>anonfun<span class="nv">$runTypelessContext$7</span>.apply<span class="o">(</span>Workflow.scala:465<span class="o">)</span> + at scala.collection.immutable.Map<span class="nv">$Map1</span>.foreach<span class="o">(</span>Map.scala:109<span class="o">)</span> + ... +</pre></td></tr></tbody></table> </div> <p>You can specify the <code>--skip-sanity-check</code> option to turn off sanityCheck:</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1</pre></td><td class="code"><pre>pio train --stop-after-read --skip-sanity-check +</pre></td></tr></tbody></table> </div> <p>You should see the checking is skipped such as the following output:</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4</pre></td><td class="code"><pre><span class="o">[</span>INFO] <span class="o">[</span>CoreWorkflow<span class="nv">$]</span> Data sanity checking is off. +<span class="o">[</span>INFO] <span class="o">[</span>CoreWorkflow<span class="nv">$]</span> Data Source +... +<span class="o">[</span>INFO] <span class="o">[</span>CoreWorkflow<span class="nv">$]</span> Training interrupted by org.apache.predictionio.workflow.StopAfterReadInterruption. +</pre></td></tr></tbody></table> </div> <h2 id='engine-status-page' class='header-anchors'>Engine Status Page</h2><p>After run <code>pio deploy</code>, you can access the engine status page by go to same URL and port of the deployed engine with your browser, which is "<a href="http://localhost:8000";>http://localhost:8000</a>" by default. In the engine status page, you can find the Engine information, and parameters of each DASE components. In particular, you can also see the "Model" trained by the algorithm based on how <code>toString()</code> method is implemented in the Algorithm's Model class.</p><h2 id='pio-shell' class='header-anchors'>pio-shell</h2><p>Apache PredictionIO (incubating) also provides <code>pio-shell</code> in which you can easily access Apache PredictionIO (incubating) API, Spark context and Spark API for quickly testing code or debugging purposes.</p><p>To bring up the shell, simply run:</p><div class="highlight shell"><table style="borde r-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1</pre></td><td class="code"><pre><span class="gp">$ </span>pio-shell --with-spark +</pre></td></tr></tbody></table> </div> <p>(<code>pio-shell</code> is available inside <code>bin/</code> directory of installed Apache PredictionIO (incubating) directory, you should be able to access it if you have added PredictionIO/bin into your environment variable <code>PATH</code>)</p><p>Note that the Spark context is available as variable <code>sc</code> inside the shell.</p><p>For example, to get the events of <code>MyApp1</code> using PEventStore API inside the pio-shell and collect them into an array <code>c</code>. run the following in the shell:</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3</pre></td><td class="code"><pre><span class="gp">> </span>import org.apache.predictionio.data.store.PEventStore +<span class="gp">> </span>val eventsRDD <span class="o">=</span> PEventStore.find<span class="o">(</span><span class="nv">appName</span><span class="o">=</span><span class="s2">"MyApp1"</span><span class="o">)(</span>sc<span class="o">)</span> +<span class="gp">> </span>val c <span class="o">=</span> eventsRDD.collect<span class="o">()</span> +</pre></td></tr></tbody></table> </div> <p>Then you should see following returned in the shell:</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3</pre></td><td class="code"><pre>... +15/05/18 14:24:42 INFO DAGScheduler: Job 0 finished: collect at <console>:24, took 1.850779 s +c: Array[org.apache.predictionio.data.storage.Event] <span class="o">=</span> Array<span class="o">(</span>Event<span class="o">(</span><span class="nv">id</span><span class="o">=</span>Some<span class="o">(</span>AaQUUBsFZxteRpDV_7fDGQAAAU1ZfRW1tX9LSWdZSb0<span class="o">)</span>,event<span class="o">=</span><span class="nv">$set</span>,eType<span class="o">=</span>item,eId<span class="o">=</span>i42,tType<span class="o">=</span>None,tId<span class="o">=</span>None,p<span class="o">=</span>DataMap<span class="o">(</span>Map<span class="o">(</span>categories -> JArray<span class="o">(</span>List<span class="o">(</span>JString<span class="o">(</span>c2<span class="o">)</span>, JString<span class="o">(</span>c1<span class="o">)</span>, JString<span class="o">(</span>c6<span class="o">)</span>, JString<span class="o">(</span>c3<span class="o">)))))</span>,t<span class="o">=</span>2015-05-15T21:31:19.349Z,tags<span class="o">=</span>List<span class="o">()</span>,pKey<span class="o">= </span>None,ct<span class="o">=</span>2015-05-15T21:31:19.354Z<span class="o">)</span>, Event<span class="o">(</span><span class="nv">id</span><span class="o">=</span>Some<span class="o">(</span>DjvP3Dnci9F4CWmiqoLabQAAAU1ZfROaqdRYO-pZ_no<span class="o">)</span>,event<span class="o">=</span><span class="nv">$set</span>,eType<span class="o">=</span>user,eId<span class="o">=</span>u9,tType<span class="o">=</span>None,tId<span class="o">=</span>None,p<span class="o">=</span>DataMap<span class="o">(</span>Map<span class="o">())</span>,t<span class="o">=</span>2015-05-15T21:31:18.810Z,tags<span class="o">=</span>List<span class="o">()</span>,pKey<span class="o">=</span>None,ct<span class="o">=</span>2015-05-15T21:31:18.817Z<span class="o">)</span>, Event<span class="o">(</span><span class="nv">id</span><span class="o">=</span>Some<span class="o">(</span>DjvP3Dnci9F4CWmiqoLabQAAAU1ZfRq7tsanlemwmZQ<span class="o">)</span>,event<span class="o">=</span>view,eType<span class="o">=</span>user, eId<span class="o">=</span>u9,tType<span class="o">=</span>Some<span class="o">(</span>item<span class="o">)</span>,tId<span class="o">=</span>Some<span class="o">(</span>i25<span class="o">)</span>,p<span class="o">=</span>DataMap<span class="o">(</span>Map<span class="o">())</span>,t<span class="o">=</span>2015-05-15T21:31:20.635Z,tags<span class="o">=</span>List<span class="o">()</span>,pKey<span class="o">=</span>None,ct<span class="o">=</span>2015-05-15T21:31:20.639Z<span class="o">)</span>, Event<span class="o">(</span><span class="nv">id</span><span class="o">=</span>Some<span class="o">(</span>DjvP3Dnci9F4CWmiqoLabQAAAU1ZfR... +</pre></td></tr></tbody></table> </div> </div></div></div></div><footer><div class="container"><div class="seperator"></div><div class="row"><div class="col-md-6 footer-link-column"><div class="footer-link-column-row"><h4>Community</h4><ul><li><a href="//predictionio.incubator.apache.org/install/" target="blank">Download</a></li><li><a href="//predictionio.incubator.apache.org/" target="blank">Docs</a></li><li><a href="//github.com/apache/incubator-predictionio" target="blank">GitHub</a></li><li><a href="mailto:[email protected]"; target="blank">Subscribe to User Mailing List</a></li><li><a href="//stackoverflow.com/questions/tagged/predictionio" target="blank">Stackoverflow</a></li></ul></div></div><div class="col-md-6 footer-link-column"><div class="footer-link-column-row"><h4>Contribute</h4><ul><li><a href="//predictionio.incubator.apache.org/community/contribute-code/" target="blank">Contribute</a></li><li><a href="//github.com/apache/incubator-pred ictionio" target="blank">Source Code</a></li><li><a href="//issues.apache.org/jira/browse/PIO" target="blank">Bug Tracker</a></li><li><a href="mailto:[email protected]"; target="blank">Subscribe to Development Mailing List</a></li></ul></div></div></div><div class="row"><div class="col-md-12 footer-link-column"><a class="pull-right" href="http://incubator.apache.org/projects/predictionio.html";><img alt="Apache Incubator" src="/images/logos/apache_incubator-6954bd16.png"/></a><span>Apache PredictionIO is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate t hat the project has yet to be fully endorsed by the ASF.</span></div></div></div><div id="footer-bottom"><div class="container"><div class="row"><div class="col-md-12"><div id="footer-logo-wrapper"><img alt="PredictionIO" src="/images/logos/logo-white-d1e9c6e6.png"/></div><div id="social-icons-wrapper"><a class="github-button" href="https://github.com/apache/incubator-predictionio"; data-style="mega" data-count-href="/apache/incubator-predictionio/stargazers" data-count-api="/repos/apache/incubator-predictionio#stargazers_count" data-count-aria-label="# stargazers on GitHub" aria-label="Star apache/incubator-predictionio on GitHub">Star</a> <a class="github-button" href="https://github.com/apache/incubator-predictionio/fork"; data-icon="octicon-git-branch" data-style="mega" data-count-href="/apache/incubator-predictionio/network" data-count-api="/repos/apache/incubator-predictionio#forks_count" data-count-aria-label="# forks on GitHub" aria-label="Fork apache/incubator-predictionio on GitHub">Fork</a> <script id="github-bjs" async="" defer="" src="https://buttons.github.io/buttons.js";></script><a href="https://twitter.com/predictionio"; target="blank"><img alt="PredictionIO on Twitter" src="/images/icons/twitter-ea9dc152.png"/></a> <a href="https://www.facebook.com/predictionio"; target="blank"><img alt="PredictionIO on Facebook" src="/images/icons/facebook-5c57939c.png"/></a> </div></div></div></div></div></footer></div><script>(function(w,d,t,u,n,s,e){w['SwiftypeObject']=n;w[n]=w[n]||function(){ +(w[n].q=w[n].q||[]).push(arguments);};s=d.createElement(t); +e=d.getElementsByTagName(t)[0];s.async=1;s.src=u;e.parentNode.insertBefore(s,e); +})(window,document,'script','//s.swiftypecdn.com/install/v1/st.js','_st'); + +_st('install','HaUfpXXV87xoB_zzCQ45');</script><script src="/javascripts/application-a6acb1f5.js"></script></body></html> \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-predictionio-site/blob/4e16ed29/customize/troubleshooting/index.html.gz ---------------------------------------------------------------------- diff --git a/customize/troubleshooting/index.html.gz b/customize/troubleshooting/index.html.gz new file mode 100644 index 0000000..88475a7 Binary files /dev/null and b/customize/troubleshooting/index.html.gz differ http://git-wip-us.apache.org/repos/asf/incubator-predictionio-site/blob/4e16ed29/datacollection/analytics-ipynb/index.html ---------------------------------------------------------------------- diff --git a/datacollection/analytics-ipynb/index.html b/datacollection/analytics-ipynb/index.html new file mode 100644 index 0000000..8184402 --- /dev/null +++ b/datacollection/analytics-ipynb/index.html @@ -0,0 +1,91 @@ +<!DOCTYPE html><html><head><title>Machine Learning Analytics with IPython Notebook</title><meta charset="utf-8"/><meta content="IE=edge,chrome=1" http-equiv="X-UA-Compatible"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta class="swiftype" name="title" data-type="string" content="Machine Learning Analytics with IPython Notebook"/><link rel="canonical" href="https://predictionio.incubator.apache.org/datacollection/analytics-ipynb/"/><link href="/images/favicon/normal-b330020a.png" rel="shortcut icon"/><link href="/images/favicon/apple-c0febcf2.png" rel="apple-touch-icon"/><link href="//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800" rel="stylesheet"/><link href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css" rel="stylesheet"/><link href="/stylesheets/application-3a3867f7.css" rel="stylesheet" type="text/css"/><script src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3. 7.2/html5shiv.min.js"></script><script src="//cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script><script src="//use.typekit.net/pqo0itb.js"></script><script>try{Typekit.load({ async: true });}catch(e){}</script></head><body><div id="global"><header><div class="container" id="header-wrapper"><div class="row"><div class="col-sm-12"><div id="logo-wrapper"><span id="drawer-toggle"></span><a href="#"></a><a href="http://predictionio.incubator.apache.org/";><img alt="PredictionIO" id="logo" src="/images/logos/logo-ee2b9bb3.png"/></a></div><div id="menu-wrapper"><div id="pill-wrapper"><a class="pill left" href="/gallery/template-gallery">TEMPLATES</a> <a class="pill right" href="//github.com/apache/incubator-predictionio/">OPEN SOURCE</a></div></div><img class="mobile-search-bar-toggler hidden-md hidden-lg" src="/images/icons/search-glass-704bd4ff.png"/></div></div></div></header><div id="search-bar-row-wrapper"><div class="container-fluid" id="search-bar-row"> <div class="row"><div class="col-md-9 col-sm-11 col-xs-11"><div class="hidden-md hidden-lg" id="mobile-page-heading-wrapper"><p>PredictionIO Docs</p><h4>Machine Learning Analytics with IPython Notebook</h4></div><h4 class="hidden-sm hidden-xs">PredictionIO Docs</h4></div><div class="col-md-3 col-sm-1 col-xs-1 hidden-md hidden-lg"><img id="left-menu-indicator" src="/images/icons/down-arrow-dfe9f7fe.png"/></div><div class="col-md-3 col-sm-12 col-xs-12 swiftype-wrapper"><div class="swiftype"><form class="search-form"><img class="search-box-toggler hidden-xs hidden-sm" src="/images/icons/search-glass-704bd4ff.png"/><div class="search-box"><img src="/images/icons/search-glass-704bd4ff.png"/><input type="text" id="st-search-input" class="st-search-input" placeholder="Search Doc..."/></div><img class="swiftype-row-hider hidden-md hidden-lg" src="/images/icons/drawer-toggle-active-fcbef12a.png"/></form></div></div><div class="mobile-left-menu-toggler hidden-md hidden-lg"></div></div></div>< /div><div id="page" class="container-fluid"><div class="row"><div id="left-menu-wrapper" class="col-md-3"><nav id="nav-main"><ul><li class="level-1"><a class="expandible" href="/"><span>Apache PredictionIO (incubating) Documentation</span></a><ul><li class="level-2"><a class="final" href="/"><span>Welcome to Apache PredictionIO (incubating)</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Getting Started</span></a><ul><li class="level-2"><a class="final" href="/start/"><span>A Quick Intro</span></a></li><li class="level-2"><a class="final" href="/install/"><span>Installing Apache PredictionIO (incubating)</span></a></li><li class="level-2"><a class="final" href="/start/download/"><span>Downloading an Engine Template</span></a></li><li class="level-2"><a class="final" href="/start/deploy/"><span>Deploying Your First Engine</span></a></li><li class="level-2"><a class="final" href="/start/customize/"><span>Customizing the Engine</span></a></li></ul></li ><li class="level-1"><a class="expandible" href="#"><span>Integrating with >Your App</span></a><ul><li class="level-2"><a class="final" >href="/appintegration/"><span>App Integration Overview</span></a></li><li >class="level-2"><a class="expandible" href="/sdk/"><span>List of >SDKs</span></a><ul><li class="level-3"><a class="final" >href="/sdk/java/"><span>Java & Android SDK</span></a></li><li >class="level-3"><a class="final" href="/sdk/php/"><span>PHP >SDK</span></a></li><li class="level-3"><a class="final" >href="/sdk/python/"><span>Python SDK</span></a></li><li class="level-3"><a >class="final" href="/sdk/ruby/"><span>Ruby SDK</span></a></li><li >class="level-3"><a class="final" href="/sdk/community/"><span>Community >Powered SDKs</span></a></li></ul></li></ul></li><li class="level-1"><a >class="expandible" href="#"><span>Deploying an Engine</span></a><ul><li >class="level-2"><a class="final" href="/deploy/"><span>Deploying as a Web >Service</span></a></li><li class="level-2"><a class="final" href="/batchpredict/"><span>Batch Predictions</span></a></li><li class="level-2"><a class="final" href="/deploy/monitoring/"><span>Monitoring Engine</span></a></li><li class="level-2"><a class="final" href="/deploy/engineparams/"><span>Setting Engine Parameters</span></a></li><li class="level-2"><a class="final" href="/deploy/enginevariants/"><span>Deploying Multiple Engine Variants</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Customizing an Engine</span></a><ul><li class="level-2"><a class="final" href="/customize/"><span>Learning DASE</span></a></li><li class="level-2"><a class="final" href="/customize/dase/"><span>Implement DASE</span></a></li><li class="level-2"><a class="final" href="/customize/troubleshooting/"><span>Troubleshooting Engine Development</span></a></li><li class="level-2"><a class="final" href="/api/current/#package"><span>Engine Scala APIs</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Coll ecting and Analyzing Data</span></a><ul><li class="level-2"><a class="final" href="/datacollection/"><span>Event Server Overview</span></a></li><li class="level-2"><a class="final" href="/datacollection/eventapi/"><span>Collecting Data with REST/SDKs</span></a></li><li class="level-2"><a class="final" href="/datacollection/eventmodel/"><span>Events Modeling</span></a></li><li class="level-2"><a class="final" href="/datacollection/webhooks/"><span>Unifying Multichannel Data with Webhooks</span></a></li><li class="level-2"><a class="final" href="/datacollection/channel/"><span>Channel</span></a></li><li class="level-2"><a class="final" href="/datacollection/batchimport/"><span>Importing Data in Batch</span></a></li><li class="level-2"><a class="final" href="/datacollection/analytics/"><span>Using Analytics Tools</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Choosing an Algorithm(s)</span></a><ul><li class="level-2"><a class="final" href="/algorithm/ "><span>Built-in Algorithm Libraries</span></a></li><li class="level-2"><a class="final" href="/algorithm/switch/"><span>Switching to Another Algorithm</span></a></li><li class="level-2"><a class="final" href="/algorithm/multiple/"><span>Combining Multiple Algorithms</span></a></li><li class="level-2"><a class="final" href="/algorithm/custom/"><span>Adding Your Own Algorithms</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>ML Tuning and Evaluation</span></a><ul><li class="level-2"><a class="final" href="/evaluation/"><span>Overview</span></a></li><li class="level-2"><a class="final" href="/evaluation/paramtuning/"><span>Hyperparameter Tuning</span></a></li><li class="level-2"><a class="final" href="/evaluation/evaluationdashboard/"><span>Evaluation Dashboard</span></a></li><li class="level-2"><a class="final" href="/evaluation/metricchoose/"><span>Choosing Evaluation Metrics</span></a></li><li class="level-2"><a class="final" href="/evaluation/metri cbuild/"><span>Building Evaluation Metrics</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>System Architecture</span></a><ul><li class="level-2"><a class="final" href="/system/"><span>Architecture Overview</span></a></li><li class="level-2"><a class="final" href="/system/anotherdatastore/"><span>Using Another Data Store</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>PredictionIO Official Templates</span></a><ul><li class="level-2"><a class="final" href="/templates/"><span>Intro</span></a></li><li class="level-2"><a class="expandible" href="#"><span>Recommendation</span></a><ul><li class="level-3"><a class="final" href="/templates/recommendation/quickstart/"><span>Quick Start</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/dase/"><span>DASE</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/evaluation/"><span>Evaluation Explained</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/how-to/"><span>How-To</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/reading-custom-events/"><span>Read Custom Events</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/customize-data-prep/"><span>Customize Data Preparator</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/customize-serving/"><span>Customize Serving</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/training-with-implicit-preference/"><span>Train with Implicit Preference</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/blacklist-items/"><span>Filter Recommended Items by Blacklist in Query</span></a></li><li class="level-3"><a class="final" href="/templates/recommendation/batch-evaluator/"><span>Batch Persistable Evaluator</span></a></li></ul></li><li class="level-2"><a class="expandi ble" href="#"><span>E-Commerce Recommendation</span></a><ul><li class="level-3"><a class="final" href="/templates/ecommercerecommendation/quickstart/"><span>Quick Start</span></a></li><li class="level-3"><a class="final" href="/templates/ecommercerecommendation/dase/"><span>DASE</span></a></li><li class="level-3"><a class="final" href="/templates/ecommercerecommendation/how-to/"><span>How-To</span></a></li><li class="level-3"><a class="final" href="/templates/ecommercerecommendation/train-with-rate-event/"><span>Train with Rate Event</span></a></li><li class="level-3"><a class="final" href="/templates/ecommercerecommendation/adjust-score/"><span>Adjust Score</span></a></li></ul></li><li class="level-2"><a class="expandible" href="#"><span>Similar Product</span></a><ul><li class="level-3"><a class="final" href="/templates/similarproduct/quickstart/"><span>Quick Start</span></a></li><li class="level-3"><a class="final" href="/templates/similarproduct/dase/"><span>DASE</span></a></li>< li class="level-3"><a class="final" href="/templates/similarproduct/how-to/"><span>How-To</span></a></li><li class="level-3"><a class="final" href="/templates/similarproduct/multi-events-multi-algos/"><span>Multiple Events and Multiple Algorithms</span></a></li><li class="level-3"><a class="final" href="/templates/similarproduct/return-item-properties/"><span>Returns Item Properties</span></a></li><li class="level-3"><a class="final" href="/templates/similarproduct/train-with-rate-event/"><span>Train with Rate Event</span></a></li><li class="level-3"><a class="final" href="/templates/similarproduct/rid-user-set-event/"><span>Get Rid of Events for Users</span></a></li><li class="level-3"><a class="final" href="/templates/similarproduct/recommended-user/"><span>Recommend Users</span></a></li></ul></li><li class="level-2"><a class="expandible" href="#"><span>Classification</span></a><ul><li class="level-3"><a class="final" href="/templates/classification/quickstart/"><span>Quick Start< /span></a></li><li class="level-3"><a class="final" href="/templates/classification/dase/"><span>DASE</span></a></li><li class="level-3"><a class="final" href="/templates/classification/how-to/"><span>How-To</span></a></li><li class="level-3"><a class="final" href="/templates/classification/add-algorithm/"><span>Use Alternative Algorithm</span></a></li><li class="level-3"><a class="final" href="/templates/classification/reading-custom-properties/"><span>Read Custom Properties</span></a></li></ul></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Engine Template Gallery</span></a><ul><li class="level-2"><a class="final" href="/gallery/template-gallery/"><span>Browse</span></a></li><li class="level-2"><a class="final" href="/community/submit-template/"><span>Submit your Engine as a Template</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Demo Tutorials</span></a><ul><li class="level-2"><a class="final" href="/demo/tapster/"><span> Comics Recommendation Demo</span></a></li><li class="level-2"><a class="final" href="/demo/community/"><span>Community Contributed Demo</span></a></li><li class="level-2"><a class="final" href="/demo/textclassification/"><span>Text Classification Engine Tutorial</span></a></li></ul></li><li class="level-1"><a class="expandible" href="/community/"><span>Getting Involved</span></a><ul><li class="level-2"><a class="final" href="/community/contribute-code/"><span>Contribute Code</span></a></li><li class="level-2"><a class="final" href="/community/contribute-documentation/"><span>Contribute Documentation</span></a></li><li class="level-2"><a class="final" href="/community/contribute-sdk/"><span>Contribute a SDK</span></a></li><li class="level-2"><a class="final" href="/community/contribute-webhook/"><span>Contribute a Webhook</span></a></li><li class="level-2"><a class="final" href="/community/projects/"><span>Community Projects</span></a></li></ul></li><li class="level-1"><a class="expa ndible" href="#"><span>Getting Help</span></a><ul><li class="level-2"><a class="final" href="/resources/faq/"><span>FAQs</span></a></li><li class="level-2"><a class="final" href="/support/"><span>Support</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Resources</span></a><ul><li class="level-2"><a class="final" href="/cli/"><span>Command-line Interface</span></a></li><li class="level-2"><a class="final" href="/resources/release/"><span>Release Cadence</span></a></li><li class="level-2"><a class="final" href="/resources/intellij/"><span>Developing Engines with IntelliJ IDEA</span></a></li><li class="level-2"><a class="final" href="/resources/upgrade/"><span>Upgrade Instructions</span></a></li><li class="level-2"><a class="final" href="/resources/glossary/"><span>Glossary</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Apache Software Foundation</span></a><ul><li class="level-2"><a class="final" href="https://www.apac he.org/"><span>Apache Homepage</span></a></li><li class="level-2"><a class="final" href="https://www.apache.org/licenses/";><span>License</span></a></li><li class="level-2"><a class="final" href="https://www.apache.org/foundation/sponsorship.html";><span>Sponsorship</span></a></li><li class="level-2"><a class="final" href="https://www.apache.org/foundation/thanks.html";><span>Thanks</span></a></li><li class="level-2"><a class="final" href="https://www.apache.org/security/";><span>Security</span></a></li></ul></li></ul></nav></div><div class="col-md-9 col-sm-12"><div class="content-header hidden-md hidden-lg"><div id="page-title"><h1>Machine Learning Analytics with IPython Notebook</h1></div></div><div id="table-of-content-wrapper"><h5>On this page</h5><aside id="table-of-contents"><ul> <li> <a href="#prerequisites">Prerequisites</a> </li> <li> <a href="#preparing-ipython-notebook">Preparing IPython Notebook</a> </li> <li> <a href="#performing-analysis-with-spark-sql">Performing Analysis with Spark SQL</a> </li> </ul> </aside><hr/><a id="edit-page-link" href="https://github.com/apache/incubator-predictionio/tree/livedoc/docs/manual/source/datacollection/analytics-ipynb.html.md.erb";><img src="/images/icons/edit-pencil-d6c1bb3d.png"/>Edit this page</a></div><div class="content-header hidden-sm hidden-xs"><div id="page-title"><h1>Machine Learning Analytics with IPython Notebook</h1></div></div><div class="content"> <p><a href="http://ipython.org/notebook.html";>IPython Notebook</a> is a very powerful interactive computational environment, and with <a href="http://predictionio.incubator.apache.org";>Apache PredictionIO (incubating)</a>, <a href="http://spark.apache.org/docs/latest/api/python/";>PySpark</a> and <a href="https://spark.apache.org/sql/";>Spark SQL</a>, you can easily analyze your collected events when you are developing or tuning your engine.</p><h2 id='prerequisites' class='header-anchors'>Prerequisites</h2><p>Before you begin, please make sure you have the l atest stable IPython installed, and that the command <code>ipython</code> can be accessed from your shell's search path.</p> <p><h2 id='export-events-to-apache-parquet' class='header-anchors'>Export Events to Apache Parquet</h2><p>PredictionIO supports exporting your events to <a href="http://parquet.incubator.apache.org/";>Apache Parquet</a>, a columnar storage format that allows you to query quickly.</p><p>Let's export the data we imported in <a href="/templates/recommendation/quickstart/#import-sample-data">Recommendation Engine Template Quick Start</a>, and assume the App ID is 1.</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1</pre></td><td class="code"><pre><span class="gp">$ </span><span class="nv">$PIO_HOME</span>/bin/pio <span class="nb">export</span> --appid 1 --output /tmp/movies --format parquet +</pre></td></tr></tbody></table> </div> <p>After the command has finished successfully, you should see something similar to the following.</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11</pre></td><td class="code"><pre>root + |-- creationTime: string <span class="o">(</span>nullable <span class="o">=</span> <span class="nb">true</span><span class="o">)</span> + |-- entityId: string <span class="o">(</span>nullable <span class="o">=</span> <span class="nb">true</span><span class="o">)</span> + |-- entityType: string <span class="o">(</span>nullable <span class="o">=</span> <span class="nb">true</span><span class="o">)</span> + |-- event: string <span class="o">(</span>nullable <span class="o">=</span> <span class="nb">true</span><span class="o">)</span> + |-- eventId: string <span class="o">(</span>nullable <span class="o">=</span> <span class="nb">true</span><span class="o">)</span> + |-- eventTime: string <span class="o">(</span>nullable <span class="o">=</span> <span class="nb">true</span><span class="o">)</span> + |-- properties: struct <span class="o">(</span>nullable <span class="o">=</span> <span class="nb">true</span><span class="o">)</span> + | |-- rating: double <span class="o">(</span>nullable <span class="o">=</span> <span class="nb">true</span><span class="o">)</span> + |-- targetEntityId: string <span class="o">(</span>nullable <span class="o">=</span> <span class="nb">true</span><span class="o">)</span> + |-- targetEntityType: string <span class="o">(</span>nullable <span class="o">=</span> <span class="nb">true</span><span class="o">)</span> +</pre></td></tr></tbody></table> </div></p><h2 id='preparing-ipython-notebook' class='header-anchors'>Preparing IPython Notebook</h2><p>Launch IPython Notebook with PySpark using the following command, with <code>$SPARK_HOME</code> replaced by the location of Apache Spark.</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1</pre></td><td class="code"><pre><span class="gp">$ </span><span class="nv">PYSPARK_DRIVER_PYTHON</span><span class="o">=</span>ipython <span class="nv">PYSPARK_DRIVER_PYTHON_OPTS</span><span class="o">=</span><span class="s2">"notebook --pylab inline"</span> <span class="nv">$SPARK_HOME</span>/bin/pyspark +</pre></td></tr></tbody></table> </div> <p>If you see a error appearing in the console like this:</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2</pre></td><td class="code"><pre><span class="o">[</span>E 10:07:53.900 NotebookApp] Support <span class="k">for </span>specifying --pylab on the <span class="nb">command </span>line has been removed. +<span class="o">[</span>E 10:07:53.901 NotebookApp] Please use <span class="sb">`</span>%pylab inline<span class="sb">`</span> or <span class="sb">`</span>%matplotlib inline<span class="sb">`</span> <span class="k">in </span>the notebook itself. +</pre></td></tr></tbody></table> </div> <p>Then you can use the following command. </p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1</pre></td><td class="code"><pre><span class="nv">PYSPARK_DRIVER_PYTHON</span><span class="o">=</span>ipython <span class="nv">PYSPARK_DRIVER_PYTHON_OPTS</span><span class="o">=</span><span class="s2">"notebook --</span><span class="sb">`</span>%pylab inline<span class="sb">`</span><span class="s2">"</span> <span class="nv">$SPARK_HOME</span>/bin/pyspark +</pre></td></tr></tbody></table> </div> <p>By default, you should be able to access your IPython Notebook via web browser at <a href="http://localhost:8888";>http://localhost:8888</a>.</p><p>Let's initialize our notebook for the following code in the first cell.</p><div class="highlight python"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4 +5 +6 +7</pre></td><td class="code"><pre><span class="kn">import</span> <span class="nn">pandas</span> <span class="kn">as</span> <span class="nn">pd</span> +<span class="k">def</span> <span class="nf">rows_to_df</span><span class="p">(</span><span class="n">rows</span><span class="p">):</span> + <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">e</span><span class="p">:</span> <span class="n">e</span><span class="o">.</span><span class="n">asDict</span><span class="p">(),</span> <span class="n">rows</span><span class="p">))</span> +<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="kn">import</span> <span class="n">SQLContext</span> +<span class="n">sqlc</span> <span class="o">=</span> <span class="n">SQLContext</span><span class="p">(</span><span class="n">sc</span><span class="p">)</span> +<span class="n">rdd</span> <span class="o">=</span> <span class="n">sqlc</span><span class="o">.</span><span class="n">parquetFile</span><span class="p">(</span><span class="s">"/tmp/movies"</span><span class="p">)</span> +<span class="n">rdd</span><span class="o">.</span><span class="n">registerTempTable</span><span class="p">(</span><span class="s">"events"</span><span class="p">)</span> +</pre></td></tr></tbody></table> </div> <p><img alt="Initialization for IPython Notebook" src="/images/datacollection/ipynb-01-004d791e.png"/></p><p><code>rows_to_df(rows)</code> will come in handy when we want to dump the results from Spark SQL using IPython Notebook's native table rendering.</p><h2 id='performing-analysis-with-spark-sql' class='header-anchors'>Performing Analysis with Spark SQL</h2><p>If all steps above ran successfully, you should have a ready-to-use analytics environment by now. Let's try a few examples to see if everything is functional.</p><p>In the second cell, put in this piece of code and run it.</p><div class="highlight python"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4 +5</pre></td><td class="code"><pre><span class="n">summary</span> <span class="o">=</span> <span class="n">sqlc</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s">"SELECT "</span> + <span class="s">"entityType, event, targetEntityType, COUNT(*) AS c "</span> + <span class="s">"FROM events "</span> + <span class="s">"GROUP BY entityType, event, targetEntityType"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span> +<span class="n">rows_to_df</span><span class="p">(</span><span class="n">summary</span><span class="p">)</span> +</pre></td></tr></tbody></table> </div> <p>You should see the following screen.</p><p><img alt="Summary of Events" src="/images/datacollection/ipynb-02-cd8b12e4.png"/></p><p>We can also plot our data, in the next two cells.</p><div class="highlight python"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4 +5 +6 +7</pre></td><td class="code"><pre><span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="kn">as</span> <span class="nn">plt</span> +<span class="n">count</span> <span class="o">=</span> <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">e</span><span class="p">:</span> <span class="n">e</span><span class="o">.</span><span class="n">c</span><span class="p">,</span> <span class="n">summary</span><span class="p">)</span> +<span class="n">event</span> <span class="o">=</span> <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">e</span><span class="p">:</span> <span class="s">"</span><span class="si">%</span><span class="s">s (</span><span class="si">%</span><span class="s">d)"</span> <span class="o">%</span> <span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">event</span><span class="p">,</span> <span class="n">e</span><span class="o">.</span><span class="n">c</span><span class="p">),</span> <span class="n">summary</span><span class="p">)</span> +<span class="n">colors</span> <span class="o">=</span> <span class="p">[</span><span class="s">'gold'</span><span class="p">,</span> <span class="s">'lightskyblue'</span><span class="p">]</span> +<span class="n">plt</span><span class="o">.</span><span class="n">pie</span><span class="p">(</span><span class="n">count</span><span class="p">,</span> <span class="n">labels</span><span class="o">=</span><span class="n">event</span><span class="p">,</span> <span class="n">colors</span><span class="o">=</span><span class="n">colors</span><span class="p">,</span> <span class="n">startangle</span><span class="o">=</span><span class="mi">90</span><span class="p">,</span> <span class="n">autopct</span><span class="o">=</span><span class="s">"</span><span class="si">%1.1</span><span class="s">f</span><span class="si">%%</span><span class="s">"</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">axis</span><span class="p">(</span><span class="s">'equal'</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span> +</pre></td></tr></tbody></table> </div> <p><img alt="Summary in Pie Chart" src="/images/datacollection/ipynb-03-28f3aa3d.png"/></p><div class="highlight python"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12</pre></td><td class="code"><pre><span class="n">ratings</span> <span class="o">=</span> <span class="n">sqlc</span><span class="o">.</span><span class="n">sql</span><span class="p">(</span><span class="s">"SELECT properties.rating AS r, COUNT(*) AS c "</span> + <span class="s">"FROM events "</span> + <span class="s">"WHERE properties.rating IS NOT NULL "</span> + <span class="s">"GROUP BY properties.rating "</span> + <span class="s">"ORDER BY r"</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span> +<span class="n">count</span> <span class="o">=</span> <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">e</span><span class="p">:</span> <span class="n">e</span><span class="o">.</span><span class="n">c</span><span class="p">,</span> <span class="n">ratings</span><span class="p">)</span> +<span class="n">rating</span> <span class="o">=</span> <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">e</span><span class="p">:</span> <span class="s">"</span><span class="si">%</span><span class="s">s (</span><span class="si">%</span><span class="s">d)"</span> <span class="o">%</span> <span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">r</span><span class="p">,</span> <span class="n">e</span><span class="o">.</span><span class="n">c</span><span class="p">),</span> <span class="n">ratings</span><span class="p">)</span> +<span class="n">colors</span> <span class="o">=</span> <span class="p">[</span><span class="s">'yellowgreen'</span><span class="p">,</span> <span class="s">'plum'</span><span class="p">,</span> <span class="s">'gold'</span><span class="p">,</span> <span class="s">'lightskyblue'</span><span class="p">,</span> <span class="s">'lightcoral'</span><span class="p">]</span> +<span class="n">plt</span><span class="o">.</span><span class="n">pie</span><span class="p">(</span><span class="n">count</span><span class="p">,</span> <span class="n">labels</span><span class="o">=</span><span class="n">rating</span><span class="p">,</span> <span class="n">colors</span><span class="o">=</span><span class="n">colors</span><span class="p">,</span> <span class="n">startangle</span><span class="o">=</span><span class="mi">90</span><span class="p">,</span> + <span class="n">autopct</span><span class="o">=</span><span class="s">"</span><span class="si">%1.1</span><span class="s">f</span><span class="si">%%</span><span class="s">"</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">axis</span><span class="p">(</span><span class="s">'equal'</span><span class="p">)</span> +<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span> +</pre></td></tr></tbody></table> </div> <p><img alt="Breakdown of Ratings" src="/images/datacollection/ipynb-04-797d73f1.png"/></p><p>Happy analyzing!</p></div></div></div></div><footer><div class="container"><div class="seperator"></div><div class="row"><div class="col-md-6 footer-link-column"><div class="footer-link-column-row"><h4>Community</h4><ul><li><a href="//predictionio.incubator.apache.org/install/" target="blank">Download</a></li><li><a href="//predictionio.incubator.apache.org/" target="blank">Docs</a></li><li><a href="//github.com/apache/incubator-predictionio" target="blank">GitHub</a></li><li><a href="mailto:[email protected]"; target="blank">Subscribe to User Mailing List</a></li><li><a href="//stackoverflow.com/questions/tagged/predictionio" target="blank">Stackoverflow</a></li></ul></div></div><div class="col-md-6 footer-link-column"><div class="footer-link-column-row"><h4>Contribute</h4><ul><li><a href="//predictionio.incubator.apache .org/community/contribute-code/" target="blank">Contribute</a></li><li><a href="//github.com/apache/incubator-predictionio" target="blank">Source Code</a></li><li><a href="//issues.apache.org/jira/browse/PIO" target="blank">Bug Tracker</a></li><li><a href="mailto:[email protected]"; target="blank">Subscribe to Development Mailing List</a></li></ul></div></div></div><div class="row"><div class="col-md-12 footer-link-column"><a class="pull-right" href="http://incubator.apache.org/projects/predictionio.html";><img alt="Apache Incubator" src="/images/logos/apache_incubator-6954bd16.png"/></a><span>Apache PredictionIO is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.</span></div></div></div><div id="footer-bottom"><div class="container"><div class="row"><div class="col-md-12"><div id="footer-logo-wrapper"><img alt="PredictionIO" src="/images/logos/logo-white-d1e9c6e6.png"/></div><div id="social-icons-wrapper"><a class="github-button" href="https://github.com/apache/incubator-predictionio"; data-style="mega" data-count-href="/apache/incubator-predictionio/stargazers" data-count-api="/repos/apache/incubator-predictionio#stargazers_count" data-count-aria-label="# stargazers on GitHub" aria-label="Star apache/incubator-predictionio on GitHub">Star</a> <a class="github-button" href="https://github.com/apache/incubator-predictionio/fork"; data-icon="octicon-git-branch" data-style="mega" data-count-href="/apache/incubator-predictionio/network" data-count-api="/repos/apache/incubator-pre dictionio#forks_count" data-count-aria-label="# forks on GitHub" aria-label="Fork apache/incubator-predictionio on GitHub">Fork</a> <script id="github-bjs" async="" defer="" src="https://buttons.github.io/buttons.js";></script><a href="https://twitter.com/predictionio"; target="blank"><img alt="PredictionIO on Twitter" src="/images/icons/twitter-ea9dc152.png"/></a> <a href="https://www.facebook.com/predictionio"; target="blank"><img alt="PredictionIO on Facebook" src="/images/icons/facebook-5c57939c.png"/></a> </div></div></div></div></div></footer></div><script>(function(w,d,t,u,n,s,e){w['SwiftypeObject']=n;w[n]=w[n]||function(){ +(w[n].q=w[n].q||[]).push(arguments);};s=d.createElement(t); +e=d.getElementsByTagName(t)[0];s.async=1;s.src=u;e.parentNode.insertBefore(s,e); +})(window,document,'script','//s.swiftypecdn.com/install/v1/st.js','_st'); + +_st('install','HaUfpXXV87xoB_zzCQ45');</script><script src="/javascripts/application-a6acb1f5.js"></script></body></html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-predictionio-site/blob/4e16ed29/datacollection/analytics-ipynb/index.html.gz ---------------------------------------------------------------------- diff --git a/datacollection/analytics-ipynb/index.html.gz b/datacollection/analytics-ipynb/index.html.gz new file mode 100644 index 0000000..9aadfae Binary files /dev/null and b/datacollection/analytics-ipynb/index.html.gz differ
