http://git-wip-us.apache.org/repos/asf/incubator-predictionio-site/blob/25938169/templates/recommendation/batch-evaluator/index.html ---------------------------------------------------------------------- diff --git a/templates/recommendation/batch-evaluator/index.html b/templates/recommendation/batch-evaluator/index.html new file mode 100644 index 0000000..84b4fea --- /dev/null +++ b/templates/recommendation/batch-evaluator/index.html @@ -0,0 +1,297 @@ +<!DOCTYPE html><html><head><title>Batch Persistable Evaluator (Recommendation)</title><meta charset="utf-8"/><meta content="IE=edge,chrome=1" http-equiv="X-UA-Compatible"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta class="swiftype" name="title" data-type="string" content="Batch Persistable Evaluator (Recommendation)"/><link rel="canonical" href="https://docs.prediction.io/templates/recommendation/batch-evaluator/"/><link href="/images/favicon/normal-b330020a.png" rel="shortcut icon"/><link href="/images/favicon/apple-c0febcf2.png" rel="apple-touch-icon"/><link href="//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800" rel="stylesheet"/><link href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css" rel="stylesheet"/><link href="/stylesheets/application-a2a2f408.css" rel="stylesheet" type="text/css"/><script src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.2/html5shiv .min.js"></script><script src="//cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script><script src="//use.typekit.net/pqo0itb.js"></script><script>try{Typekit.load({ async: true });}catch(e){}</script></head><body><div id="global"><header><div class="container" id="header-wrapper"><div class="row"><div class="col-sm-12"><div id="logo-wrapper"><span id="drawer-toggle"></span><a href="#"></a><a href="http://predictionio.incubator.apache.org/";><img alt="PredictionIO" id="logo" src="/images/logos/logo-ee2b9bb3.png"/></a></div><div id="menu-wrapper"><div id="pill-wrapper"><a class="pill left" href="/gallery/template-gallery">TEMPLATES</a> <a class="pill right" href="//github.com/apache/incubator-predictionio/">OPEN SOURCE</a></div></div><img class="mobile-search-bar-toggler hidden-md hidden-lg" src="/images/icons/search-glass-704bd4ff.png"/></div></div></div></header><div id="search-bar-row-wrapper"><div class="container-fluid" id="search-bar-row"><div class="r ow"><div class="col-md-9 col-sm-11 col-xs-11"><div class="hidden-md hidden-lg" id="mobile-page-heading-wrapper"><p>PredictionIO Docs</p><h4>Batch Persistable Evaluator (Recommendation)</h4></div><h4 class="hidden-sm hidden-xs">PredictionIO Docs</h4></div><div class="col-md-3 col-sm-1 col-xs-1 hidden-md hidden-lg"><img id="left-menu-indicator" src="/images/icons/down-arrow-dfe9f7fe.png"/></div><div class="col-md-3 col-sm-12 col-xs-12 swiftype-wrapper"><div class="swiftype"><form class="search-form"><img class="search-box-toggler hidden-xs hidden-sm" src="/images/icons/search-glass-704bd4ff.png"/><div class="search-box"><img src="/images/icons/search-glass-704bd4ff.png"/><input type="text" id="st-search-input" class="st-search-input" placeholder="Search Doc..."/></div><img class="swiftype-row-hider hidden-md hidden-lg" src="/images/icons/drawer-toggle-active-fcbef12a.png"/></form></div></div><div class="mobile-left-menu-toggler hidden-md hidden-lg"></div></div></div></div><div id="pag e" class="container-fluid"><div class="row"><div id="left-menu-wrapper" class="col-md-3"><nav id="nav-main"><ul><li class="level-1"><a class="expandible" href="/"><span>Apache PredictionIO (incubating) Documentation</span></a><ul><li class="level-2"><a class="final" href="/"><span>Welcome to Apache PredictionIO (incubating)</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Getting Started</span></a><ul><li class="level-2"><a class="final" href="/start/"><span>A Quick Intro</span></a></li><li class="level-2"><a class="final" href="/install/"><span>Installing Apache PredictionIO (incubating)</span></a></li><li class="level-2"><a class="final" href="/start/download/"><span>Downloading an Engine Template</span></a></li><li class="level-2"><a class="final" href="/start/deploy/"><span>Deploying Your First Engine</span></a></li><li class="level-2"><a class="final" href="/start/customize/"><span>Customizing the Engine</span></a></li></ul></li><li class="level -1"><a class="expandible" href="#"><span>Integrating with Your App</span></a><ul><li class="level-2"><a class="final" href="/appintegration/"><span>App Integration Overview</span></a></li><li class="level-2"><a class="expandible" href="/sdk/"><span>List of SDKs</span></a><ul><li class="level-3"><a class="final" href="/sdk/java/"><span>Java & Android SDK</span></a></li><li class="level-3"><a class="final" href="/sdk/php/"><span>PHP SDK</span></a></li><li class="level-3"><a class="final" href="/sdk/python/"><span>Python SDK</span></a></li><li class="level-3"><a class="final" href="/sdk/ruby/"><span>Ruby SDK</span></a></li><li class="level-3"><a class="final" href="/sdk/community/"><span>Community Powered SDKs</span></a></li></ul></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Deploying an Engine</span></a><ul><li class="level-2"><a class="final" href="/deploy/"><span>Deploying as a Web Service</span></a></li><li class="level-2"><a class="final" href="/cli/#engi ne-commands"><span>Engine Command-line Interface</span></a></li><li class="level-2"><a class="final" href="/deploy/monitoring/"><span>Monitoring Engine</span></a></li><li class="level-2"><a class="final" href="/deploy/engineparams/"><span>Setting Engine Parameters</span></a></li><li class="level-2"><a class="final" href="/deploy/enginevariants/"><span>Deploying Multiple Engine Variants</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Customizing an Engine</span></a><ul><li class="level-2"><a class="final" href="/customize/"><span>Learning DASE</span></a></li><li class="level-2"><a class="final" href="/customize/dase/"><span>Implement DASE</span></a></li><li class="level-2"><a class="final" href="/customize/troubleshooting/"><span>Troubleshooting Engine Development</span></a></li><li class="level-2"><a class="final" href="/api/current/#package"><span>Engine Scala APIs</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Co llecting and Analyzing Data</span></a><ul><li class="level-2"><a class="final" href="/datacollection/"><span>Event Server Overview</span></a></li><li class="level-2"><a class="final" href="/cli/#event-server-commands"><span>Event Server Command-line Interface</span></a></li><li class="level-2"><a class="final" href="/datacollection/eventapi/"><span>Collecting Data with REST/SDKs</span></a></li><li class="level-2"><a class="final" href="/datacollection/eventmodel/"><span>Events Modeling</span></a></li><li class="level-2"><a class="final" href="/datacollection/webhooks/"><span>Unifying Multichannel Data with Webhooks</span></a></li><li class="level-2"><a class="final" href="/datacollection/channel/"><span>Channel</span></a></li><li class="level-2"><a class="final" href="/datacollection/batchimport/"><span>Importing Data in Batch</span></a></li><li class="level-2"><a class="final" href="/datacollection/analytics/"><span>Using Analytics Tools</span></a></li></ul></li><li class="level-1" ><a class="expandible" href="#"><span>Choosing an >Algorithm(s)</span></a><ul><li class="level-2"><a class="final" >href="/algorithm/"><span>Built-in Algorithm Libraries</span></a></li><li >class="level-2"><a class="final" href="/algorithm/switch/"><span>Switching to >Another Algorithm</span></a></li><li class="level-2"><a class="final" >href="/algorithm/multiple/"><span>Combining Multiple >Algorithms</span></a></li><li class="level-2"><a class="final" >href="/algorithm/custom/"><span>Adding Your Own >Algorithms</span></a></li></ul></li><li class="level-1"><a class="expandible" >href="#"><span>ML Tuning and Evaluation</span></a><ul><li class="level-2"><a >class="final" href="/evaluation/"><span>Overview</span></a></li><li >class="level-2"><a class="final" >href="/evaluation/paramtuning/"><span>Hyperparameter >Tuning</span></a></li><li class="level-2"><a class="final" >href="/evaluation/evaluationdashboard/"><span>Evaluation >Dashboard</span></a></li><li class="level-2"><a class="final" href="/eval uation/metricchoose/"><span>Choosing Evaluation Metrics</span></a></li><li class="level-2"><a class="final" href="/evaluation/metricbuild/"><span>Building Evaluation Metrics</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>System Architecture</span></a><ul><li class="level-2"><a class="final" href="/system/"><span>Architecture Overview</span></a></li><li class="level-2"><a class="final" href="/system/anotherdatastore/"><span>Using Another Data Store</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Engine Template Gallery</span></a><ul><li class="level-2"><a class="final" href="/gallery/template-gallery/"><span>Browse</span></a></li><li class="level-2"><a class="final" href="/community/submit-template/"><span>Submit your Engine as a Template</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Demo Tutorials</span></a><ul><li class="level-2"><a class="final" href="/demo/tapster/"><span>Comi cs Recommendation Demo</span></a></li><li class="level-2"><a class="final" href="/demo/community/"><span>Community Contributed Demo</span></a></li><li class="level-2"><a class="final" href="/demo/textclassification/"><span>Text Classification Engine Tutorial</span></a></li></ul></li><li class="level-1"><a class="expandible" href="/community/"><span>Getting Involved</span></a><ul><li class="level-2"><a class="final" href="/community/contribute-code/"><span>Contribute Code</span></a></li><li class="level-2"><a class="final" href="/community/contribute-documentation/"><span>Contribute Documentation</span></a></li><li class="level-2"><a class="final" href="/community/contribute-sdk/"><span>Contribute a SDK</span></a></li><li class="level-2"><a class="final" href="/community/contribute-webhook/"><span>Contribute a Webhook</span></a></li><li class="level-2"><a class="final" href="/community/projects/"><span>Community Projects</span></a></li></ul></li><li class="level-1"><a class="expandib le" href="#"><span>Getting Help</span></a><ul><li class="level-2"><a class="final" href="/resources/faq/"><span>FAQs</span></a></li><li class="level-2"><a class="final" href="/support/"><span>Support</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Resources</span></a><ul><li class="level-2"><a class="final" href="/resources/intellij/"><span>Developing Engines with IntelliJ IDEA</span></a></li><li class="level-2"><a class="final" href="/resources/upgrade/"><span>Upgrade Instructions</span></a></li><li class="level-2"><a class="final" href="/resources/glossary/"><span>Glossary</span></a></li></ul></li></ul></nav></div><div class="col-md-9 col-sm-12"><div class="content-header hidden-md hidden-lg"><div id="page-title"><h1>Batch Persistable Evaluator (Recommendation)</h1></div></div><div id="table-of-content-wrapper"><h5>On this page</h5><aside id="table-of-contents"><ul> <li> <a href="#1-modify-datasource">1. Modify DataSource</a> </li> <li> <a href="# 2-add-a-new-evaluator">2. Add a new Evaluator</a> </li> <li> <a href="#3-define-evaluation-and-engineparamsgenerator-object">3. Define Evaluation and EngineParamsGenerator object</a> </li> <li> <a href="#4-build-and-run">4. build and run</a> </li> </ul> </aside><hr/><a id="edit-page-link" href="https://github.com/apache/incubator-predictionio/tree/livedoc/docs/manual/source/templates/recommendation/batch-evaluator.html.md";><img src="/images/icons/edit-pencil-d6c1bb3d.png"/>Edit this page</a></div><div class="content-header hidden-sm hidden-xs"><div id="page-title"><h1>Batch Persistable Evaluator (Recommendation)</h1></div></div><div class="content"><p>This how-to tutorial would explain how you can also use <code>$pio eval</code> to persist predicted result for a batch of queries. Please read the <a href="/templates/recommendation/evaluation/">Evaluation</a> to understand the usage of DataSoure's <code>readEval()</code> and the Evaluation component first.</p><div class="alert-mes sage warning"><p>This tutorial is based on some experimental and developer features, which may be changed in future release.</p></div><div class="alert-message note"><p>This tutorial is based on Recommendation template version v0.3.2</p></div><h2 id='1.-modify-datasource' class='header-anchors'>1. Modify DataSource</h2><p>Modify DataSource's <code>readEval()</code> to generate the batch Queries which you want to run batch predict.</p><div class="highlight scala"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31</pre></td><td class="code"><pre> + <span class="k">override</span> + <span class="k">def</span> <span class="n">readEval</span><span class="o">(</span><span class="n">sc</span><span class="k">:</span> <span class="kt">SparkContext</span><span class="o">)</span> + <span class="k">:</span> <span class="kt">Seq</span><span class="o">[(</span><span class="kt">TrainingData</span>, <span class="kt">EmptyEvaluationInfo</span>, <span class="kt">RDD</span><span class="o">[(</span><span class="kt">Query</span>, <span class="kt">ActualResult</span><span class="o">)])]</span> <span class="k">=</span> <span class="o">{</span> + <span class="c1">// This function only return one evaluation data set +</span> + <span class="c1">// Create your own queries here. Below are provided as examples. +</span> <span class="c1">// for example, you may get all distinct user id from the trainingData to create the Query +</span> <span class="k">val</span> <span class="n">batchQueries</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[</span><span class="kt">Query</span><span class="o">]</span> <span class="k">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="o">(</span> + <span class="nc">Seq</span><span class="o">(</span> + <span class="nc">Query</span><span class="o">(</span><span class="n">user</span> <span class="k">=</span> <span class="s">"1"</span><span class="o">,</span> <span class="n">num</span> <span class="k">=</span> <span class="mi">10</span><span class="o">),</span> + <span class="nc">Query</span><span class="o">(</span><span class="n">user</span> <span class="k">=</span> <span class="s">"3"</span><span class="o">,</span> <span class="n">num</span> <span class="k">=</span> <span class="mi">15</span><span class="o">),</span> + <span class="nc">Query</span><span class="o">(</span><span class="n">user</span> <span class="k">=</span> <span class="s">"5"</span><span class="o">,</span> <span class="n">num</span> <span class="k">=</span> <span class="mi">20</span><span class="o">)</span> + <span class="o">)</span> + <span class="o">)</span> + + <span class="k">val</span> <span class="n">queryAndActual</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[(</span><span class="kt">Query</span>, <span class="kt">ActualResult</span><span class="o">)]</span> <span class="k">=</span> <span class="n">batchQueries</span><span class="o">.</span><span class="n">map</span> <span class="o">(</span><span class="n">q</span> <span class="k">=></span> + <span class="c1">// the ActualResult contain dummy empty rating array +</span> <span class="c1">// because we not interested in Actual result for batch predict purpose. +</span> <span class="o">(</span><span class="n">q</span><span class="o">,</span> <span class="nc">ActualResult</span><span class="o">(</span><span class="nc">Array</span><span class="o">()))</span> + <span class="o">)</span> + + <span class="k">val</span> <span class="n">evalDataSet</span> <span class="k">=</span> <span class="o">(</span> + <span class="n">readTraining</span><span class="o">(</span><span class="n">sc</span><span class="o">),</span> + <span class="k">new</span> <span class="nc">EmptyEvaluationInfo</span><span class="o">(),</span> + <span class="n">queryAndActual</span> + <span class="o">)</span> + + <span class="nc">Seq</span><span class="o">(</span><span class="n">evalDataSet</span><span class="o">)</span> + <span class="o">}</span> + +</pre></td></tr></tbody></table> </div> <div class="alert-message note"><p>Alternatively, you can create a new DataSource extending original DataSource. Then you can add the new one in Engine.scala and specify which one to use in engine.json.</p></div> <h2 id='2.-add-a-new-evaluator' class='header-anchors'>2. Add a new Evaluator</h2><p>Create a new file <code>BatchPersistableEvaluator.scala</code>. Unlike the <code>MetricEvaluator</code>, this Evaluator simply writes the Query and correpsonding PredictedResult to the output directory without performaning any metrics calculation.</p><p>Note that output directory is specified by the variable <code>outputDir</code>.</p><div class="highlight scala"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82</pre></td><td class="code"><pre><span class="k">package</span> <span class="nn">org.template.recommendation</span> + +<span class="k">import</span> <span class="nn">org.apache.predictionio.controller.EmptyEvaluationInfo</span> +<span class="k">import</span> <span class="nn">org.apache.predictionio.controller.Engine</span> +<span class="k">import</span> <span class="nn">org.apache.predictionio.controller.EngineParams</span> +<span class="k">import</span> <span class="nn">org.apache.predictionio.controller.EngineParamsGenerator</span> +<span class="k">import</span> <span class="nn">org.apache.predictionio.controller.Evaluation</span> +<span class="k">import</span> <span class="nn">org.apache.predictionio.controller.Params</span> +<span class="k">import</span> <span class="nn">org.apache.predictionio.core.BaseEvaluator</span> +<span class="k">import</span> <span class="nn">org.apache.predictionio.core.BaseEvaluatorResult</span> +<span class="k">import</span> <span class="nn">org.apache.predictionio.workflow.WorkflowParams</span> + +<span class="k">import</span> <span class="nn">org.apache.spark.SparkContext</span> +<span class="k">import</span> <span class="nn">org.apache.spark.rdd.RDD</span> + +<span class="k">import</span> <span class="nn">org.json4s.DefaultFormats</span> +<span class="k">import</span> <span class="nn">org.json4s.Formats</span> +<span class="k">import</span> <span class="nn">org.json4s.native.Serialization</span> + +<span class="k">import</span> <span class="nn">grizzled.slf4j.Logger</span> + +<span class="k">class</span> <span class="nc">BatchPersistableEvaluatorResult</span> <span class="k">extends</span> <span class="nc">BaseEvaluatorResult</span> <span class="o">{}</span> + +<span class="k">class</span> <span class="nc">BatchPersistableEvaluator</span> <span class="k">extends</span> <span class="nc">BaseEvaluator</span><span class="o">[</span> + <span class="kt">EmptyEvaluationInfo</span>, + <span class="kt">Query</span>, + <span class="kt">PredictedResult</span>, + <span class="kt">ActualResult</span>, + <span class="kt">BatchPersistableEvaluatorResult</span><span class="o">]</span> <span class="o">{</span> + <span class="nd">@transient</span> <span class="k">lazy</span> <span class="k">val</span> <span class="n">logger</span> <span class="k">=</span> <span class="nc">Logger</span><span class="o">[</span><span class="kt">this.</span><span class="k">type</span><span class="o">]</span> + + <span class="c1">// A helper object for the json4s serialization +</span> <span class="k">case</span> <span class="k">class</span> <span class="nc">Row</span><span class="o">(</span><span class="n">query</span><span class="k">:</span> <span class="kt">Query</span><span class="o">,</span> <span class="n">predictedResult</span><span class="k">:</span> <span class="kt">PredictedResult</span><span class="o">)</span> + <span class="k">extends</span> <span class="nc">Serializable</span> + + <span class="k">def</span> <span class="n">evaluateBase</span><span class="o">(</span> + <span class="n">sc</span><span class="k">:</span> <span class="kt">SparkContext</span><span class="o">,</span> + <span class="n">evaluation</span><span class="k">:</span> <span class="kt">Evaluation</span><span class="o">,</span> + <span class="n">engineEvalDataSet</span><span class="k">:</span> <span class="kt">Seq</span><span class="o">[(</span> + <span class="kt">EngineParams</span>, + <span class="kt">Seq</span><span class="o">[(</span><span class="kt">EmptyEvaluationInfo</span>, <span class="kt">RDD</span><span class="o">[(</span><span class="kt">Query</span>, <span class="kt">PredictedResult</span>, <span class="kt">ActualResult</span><span class="o">)])])],</span> + <span class="n">params</span><span class="k">:</span> <span class="kt">WorkflowParams</span><span class="o">)</span><span class="k">:</span> <span class="kt">BatchPersistableEvaluatorResult</span> <span class="o">=</span> <span class="o">{</span> + + <span class="cm">/** Extract the first data, as we are only interested in the first + * evaluation. It is possible to relax this restriction, and have the + * output logic below to write to different directory for different engine + * params. + */</span> + + <span class="n">require</span><span class="o">(</span> + <span class="n">engineEvalDataSet</span><span class="o">.</span><span class="n">size</span> <span class="o">==</span> <span class="mi">1</span><span class="o">,</span> <span class="s">"There should be only one engine params"</span><span class="o">)</span> + + <span class="k">val</span> <span class="n">evalDataSet</span> <span class="k">=</span> <span class="n">engineEvalDataSet</span><span class="o">.</span><span class="n">head</span><span class="o">.</span><span class="n">_2</span> + + <span class="n">require</span><span class="o">(</span><span class="n">evalDataSet</span><span class="o">.</span><span class="n">size</span> <span class="o">==</span> <span class="mi">1</span><span class="o">,</span> <span class="s">"There should be only one RDD[(Q, P, A)]"</span><span class="o">)</span> + + <span class="k">val</span> <span class="n">qpaRDD</span> <span class="k">=</span> <span class="n">evalDataSet</span><span class="o">.</span><span class="n">head</span><span class="o">.</span><span class="n">_2</span> + + <span class="c1">// qpaRDD contains 3 queries we specified in readEval, the corresponding +</span> <span class="c1">// predictedResults, and the dummy actual result. +</span> + <span class="cm">/** The output directory. Better to use absolute path if you run on cluster. + * If your database has a Hadoop interface, you can also convert the + * following to write to your database in parallel as well. + */</span> + <span class="k">val</span> <span class="n">outputDir</span> <span class="k">=</span> <span class="s">"batch_result"</span> + + <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="o">(</span><span class="s">"Writing result to disk"</span><span class="o">)</span> + <span class="n">qpaRDD</span> + <span class="o">.</span><span class="n">map</span> <span class="o">{</span> <span class="k">case</span> <span class="o">(</span><span class="n">q</span><span class="o">,</span> <span class="n">p</span><span class="o">,</span> <span class="n">a</span><span class="o">)</span> <span class="k">=></span> <span class="nc">Row</span><span class="o">(</span><span class="n">q</span><span class="o">,</span> <span class="n">p</span><span class="o">)</span> <span class="o">}</span> + <span class="o">.</span><span class="n">map</span> <span class="o">{</span> <span class="n">row</span> <span class="k">=></span> + <span class="c1">// Convert into a json +</span> <span class="k">implicit</span> <span class="k">val</span> <span class="n">formats</span><span class="k">:</span> <span class="kt">Formats</span> <span class="o">=</span> <span class="nc">DefaultFormats</span> + <span class="nc">Serialization</span><span class="o">.</span><span class="n">write</span><span class="o">(</span><span class="n">row</span><span class="o">)</span> + <span class="o">}</span> + <span class="o">.</span><span class="n">saveAsTextFile</span><span class="o">(</span><span class="n">outputDir</span><span class="o">)</span> + + <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="o">(</span><span class="n">s</span><span class="s">"Result can be found in $outputDir"</span><span class="o">)</span> + + <span class="k">new</span> <span class="nc">BatchPersistableEvaluatorResult</span><span class="o">()</span> + <span class="o">}</span> +<span class="o">}</span> +</pre></td></tr></tbody></table> </div> <h2 id='3.-define-evaluation-and-engineparamsgenerator-object' class='header-anchors'>3. Define Evaluation and EngineParamsGenerator object</h2><p>Create a new file <code>BatchEvaluation.scala</code>. Note that the new <code>BatchPersistableEvaluator</code> is used. The <code>BatchEngineParamsList</code> specifies the parameters of the engine.</p><p>Modify the appName parameter in <code>DataSourceParams</code> to match your app name.</p><div class="highlight scala"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29</pre></td><td class="code"><pre><span class="k">package</span> <span class="nn">org.template.recommendation</span> + +<span class="k">import</span> <span class="nn">org.apache.predictionio.controller.EngineParamsGenerator</span> +<span class="k">import</span> <span class="nn">org.apache.predictionio.controller.EngineParams</span> +<span class="k">import</span> <span class="nn">org.apache.predictionio.controller.Evaluation</span> + +<span class="k">object</span> <span class="nc">BatchEvaluation</span> <span class="k">extends</span> <span class="nc">Evaluation</span> <span class="o">{</span> + <span class="c1">// Define Engine and Evaluator used in Evaluation +</span> + <span class="cm">/** + * Specify the new BatchPersistableEvaluator. + */</span> + <span class="n">engineEvaluator</span> <span class="k">=</span> + <span class="o">(</span><span class="nc">RecommendationEngine</span><span class="o">(),</span> <span class="k">new</span> <span class="nc">BatchPersistableEvaluator</span><span class="o">())</span> +<span class="o">}</span> + +<span class="k">object</span> <span class="nc">BatchEngineParamsList</span> <span class="k">extends</span> <span class="nc">EngineParamsGenerator</span> <span class="o">{</span> + <span class="c1">// We only interest in a single engine params. +</span> <span class="n">engineParamsList</span> <span class="k">=</span> <span class="nc">Seq</span><span class="o">(</span> + <span class="nc">EngineParams</span><span class="o">(</span> + <span class="n">dataSourceParams</span> <span class="k">=</span> + <span class="nc">DataSourceParams</span><span class="o">(</span><span class="n">appName</span> <span class="k">=</span> <span class="s">"INVALID_APP_NAME"</span><span class="o">,</span> <span class="n">evalParams</span> <span class="k">=</span> <span class="nc">None</span><span class="o">),</span> + <span class="n">algorithmParamsList</span> <span class="k">=</span> <span class="nc">Seq</span><span class="o">((</span><span class="s">"als"</span><span class="o">,</span> <span class="nc">ALSAlgorithmParams</span><span class="o">(</span> + <span class="n">rank</span> <span class="k">=</span> <span class="mi">10</span><span class="o">,</span> + <span class="n">numIterations</span> <span class="k">=</span> <span class="mi">20</span><span class="o">,</span> + <span class="n">lambda</span> <span class="k">=</span> <span class="mf">0.01</span><span class="o">,</span> + <span class="n">seed</span> <span class="k">=</span> <span class="nc">Some</span><span class="o">(</span><span class="mi">3L</span><span class="o">))))))</span> +<span class="o">}</span> + +</pre></td></tr></tbody></table> </div> <h2 id='4.-build-and-run' class='header-anchors'>4. build and run</h2><p>Run the following command to build</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1</pre></td><td class="code"><pre><span class="gp">$ </span>pio build +</pre></td></tr></tbody></table> </div> <p>After the build is successful, you should see the following outputs:</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1</pre></td><td class="code"><pre><span class="o">[</span>INFO] <span class="o">[</span>Console<span class="nv">$]</span> Your engine is ready <span class="k">for </span>training. +</pre></td></tr></tbody></table> </div> <p>To run the <code>BatchEvaluation</code> with <code>BatchEngineParamsList</code>, run the following command:</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1</pre></td><td class="code"><pre><span class="gp">$ </span>pio <span class="nb">eval </span>org.template.recommendation.BatchEvaluation org.template.recommendation.BatchEngineParamsList +</pre></td></tr></tbody></table> </div> <p>You should see the following outputs:</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4</pre></td><td class="code"><pre><span class="o">[</span>INFO] <span class="o">[</span>BatchPersistableEvaluator] Writing result to disk +<span class="o">[</span>INFO] <span class="o">[</span>BatchPersistableEvaluator] Result can be found <span class="k">in </span>batch_result +<span class="o">[</span>INFO] <span class="o">[</span>CoreWorkflow<span class="nv">$]</span> Updating evaluation instance with result: org.template.recommendation.BatchPersistableEvaluatorResult@2f886889 +<span class="o">[</span>INFO] <span class="o">[</span>CoreWorkflow<span class="nv">$]</span> runEvaluation completed +</pre></td></tr></tbody></table> </div> <p>You should find the batch queries and the predicted results in the output directory <code>batch_result/</code>.</p></div></div></div></div><footer><div class="container"><div class="seperator"></div><div class="row"><div class="col-md-6 col-xs-6 footer-link-column"><div class="footer-link-column-row"><h4>Community</h4><ul><li><a href="//docs.prediction.io/install/" target="blank">Download</a></li><li><a href="//docs.prediction.io/" target="blank">Docs</a></li><li><a href="//github.com/apache/incubator-predictionio" target="blank">GitHub</a></li><li><a href="mailto:user-subscr...@predictionio.incubator.apache.org"; target="blank">Subscribe to User Mailing List</a></li><li><a href="//stackoverflow.com/questions/tagged/predictionio" target="blank">Stackoverflow</a></li></ul></div></div><div class="col-md-6 col-xs-6 footer-link-column"><div class="footer-link-column-row"><h4>Contribute</h4><ul><li><a href="//predictionio.incubator.apache.org/com munity/contribute-code/" target="blank">Contribute</a></li><li><a href="//github.com/apache/incubator-predictionio" target="blank">Source Code</a></li><li><a href="//issues.apache.org/jira/browse/PIO" target="blank">Bug Tracker</a></li><li><a href="mailto:dev-subscr...@predictionio.incubator.apache.org"; target="blank">Subscribe to Development Mailing List</a></li></ul></div></div></div></div><div id="footer-bottom"><div class="container"><div class="row"><div class="col-md-12"><div id="footer-logo-wrapper"><img alt="PredictionIO" src="/images/logos/logo-white-d1e9c6e6.png"/></div><div id="social-icons-wrapper"><a class="github-button" href="https://github.com/apache/incubator-predictionio"; data-style="mega" data-count-href="/apache/incubator-predictionio/stargazers" data-count-api="/repos/apache/incubator-predictionio#stargazers_count" data-count-aria-label="# stargazers on GitHub" aria-label="Star apache/incubator-predictionio on GitHub">Star</a> <a class="github-button" href="http s://github.com/apache/incubator-predictionio/fork" data-icon="octicon-git-branch" data-style="mega" data-count-href="/apache/incubator-predictionio/network" data-count-api="/repos/apache/incubator-predictionio#forks_count" data-count-aria-label="# forks on GitHub" aria-label="Fork apache/incubator-predictionio on GitHub">Fork</a> <script id="github-bjs" async="" defer="" src="https://buttons.github.io/buttons.js";></script><a href="//www.facebook.com/predictionio" target="blank"><img alt="PredictionIO on Twitter" src="/images/icons/twitter-ea9dc152.png"/></a> <a href="//twitter.com/predictionio" target="blank"><img alt="PredictionIO on Facebook" src="/images/icons/facebook-5c57939c.png"/></a> </div></div></div></div></div></footer></div><script>(function(w,d,t,u,n,s,e){w['SwiftypeObject']=n;w[n]=w[n]||function(){ +(w[n].q=w[n].q||[]).push(arguments);};s=d.createElement(t); +e=d.getElementsByTagName(t)[0];s.async=1;s.src=u;e.parentNode.insertBefore(s,e); +})(window,document,'script','//s.swiftypecdn.com/install/v1/st.js','_st'); + +_st('install','HaUfpXXV87xoB_zzCQ45');</script><script src="/javascripts/application-280db181.js"></script></body></html> \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-predictionio-site/blob/25938169/templates/recommendation/batch-evaluator/index.html.gz ---------------------------------------------------------------------- diff --git a/templates/recommendation/batch-evaluator/index.html.gz b/templates/recommendation/batch-evaluator/index.html.gz new file mode 100644 index 0000000..071fada Binary files /dev/null and b/templates/recommendation/batch-evaluator/index.html.gz differ http://git-wip-us.apache.org/repos/asf/incubator-predictionio-site/blob/25938169/templates/recommendation/blacklist-items/index.html ---------------------------------------------------------------------- diff --git a/templates/recommendation/blacklist-items/index.html b/templates/recommendation/blacklist-items/index.html new file mode 100644 index 0000000..48b3e38 --- /dev/null +++ b/templates/recommendation/blacklist-items/index.html @@ -0,0 +1,159 @@ +<!DOCTYPE html><html><head><title>Filter Recommended Items by Blacklist in Query (Recommendation)</title><meta charset="utf-8"/><meta content="IE=edge,chrome=1" http-equiv="X-UA-Compatible"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta class="swiftype" name="title" data-type="string" content="Filter Recommended Items by Blacklist in Query (Recommendation)"/><link rel="canonical" href="https://docs.prediction.io/templates/recommendation/blacklist-items/"/><link href="/images/favicon/normal-b330020a.png" rel="shortcut icon"/><link href="/images/favicon/apple-c0febcf2.png" rel="apple-touch-icon"/><link href="//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800" rel="stylesheet"/><link href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css" rel="stylesheet"/><link href="/stylesheets/application-a2a2f408.css" rel="stylesheet" type="text/css"/><script src="//cdnjs.cloudflare.c om/ajax/libs/html5shiv/3.7.2/html5shiv.min.js"></script><script src="//cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script><script src="//use.typekit.net/pqo0itb.js"></script><script>try{Typekit.load({ async: true });}catch(e){}</script></head><body><div id="global"><header><div class="container" id="header-wrapper"><div class="row"><div class="col-sm-12"><div id="logo-wrapper"><span id="drawer-toggle"></span><a href="#"></a><a href="http://predictionio.incubator.apache.org/";><img alt="PredictionIO" id="logo" src="/images/logos/logo-ee2b9bb3.png"/></a></div><div id="menu-wrapper"><div id="pill-wrapper"><a class="pill left" href="/gallery/template-gallery">TEMPLATES</a> <a class="pill right" href="//github.com/apache/incubator-predictionio/">OPEN SOURCE</a></div></div><img class="mobile-search-bar-toggler hidden-md hidden-lg" src="/images/icons/search-glass-704bd4ff.png"/></div></div></div></header><div id="search-bar-row-wrapper"><div class="container-fl uid" id="search-bar-row"><div class="row"><div class="col-md-9 col-sm-11 col-xs-11"><div class="hidden-md hidden-lg" id="mobile-page-heading-wrapper"><p>PredictionIO Docs</p><h4>Filter Recommended Items by Blacklist in Query (Recommendation)</h4></div><h4 class="hidden-sm hidden-xs">PredictionIO Docs</h4></div><div class="col-md-3 col-sm-1 col-xs-1 hidden-md hidden-lg"><img id="left-menu-indicator" src="/images/icons/down-arrow-dfe9f7fe.png"/></div><div class="col-md-3 col-sm-12 col-xs-12 swiftype-wrapper"><div class="swiftype"><form class="search-form"><img class="search-box-toggler hidden-xs hidden-sm" src="/images/icons/search-glass-704bd4ff.png"/><div class="search-box"><img src="/images/icons/search-glass-704bd4ff.png"/><input type="text" id="st-search-input" class="st-search-input" placeholder="Search Doc..."/></div><img class="swiftype-row-hider hidden-md hidden-lg" src="/images/icons/drawer-toggle-active-fcbef12a.png"/></form></div></div><div class="mobile-left-menu-toggler hidden-md hidden-lg"></div></div></div></div><div id="page" class="container-fluid"><div class="row"><div id="left-menu-wrapper" class="col-md-3"><nav id="nav-main"><ul><li class="level-1"><a class="expandible" href="/"><span>Apache PredictionIO (incubating) Documentation</span></a><ul><li class="level-2"><a class="final" href="/"><span>Welcome to Apache PredictionIO (incubating)</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Getting Started</span></a><ul><li class="level-2"><a class="final" href="/start/"><span>A Quick Intro</span></a></li><li class="level-2"><a class="final" href="/install/"><span>Installing Apache PredictionIO (incubating)</span></a></li><li class="level-2"><a class="final" href="/start/download/"><span>Downloading an Engine Template</span></a></li><li class="level-2"><a class="final" href="/start/deploy/"><span>Deploying Your First Engine</span></a></li><li class="level-2"><a class="final" href="/start/customize/"><span>Customi zing the Engine</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Integrating with Your App</span></a><ul><li class="level-2"><a class="final" href="/appintegration/"><span>App Integration Overview</span></a></li><li class="level-2"><a class="expandible" href="/sdk/"><span>List of SDKs</span></a><ul><li class="level-3"><a class="final" href="/sdk/java/"><span>Java & Android SDK</span></a></li><li class="level-3"><a class="final" href="/sdk/php/"><span>PHP SDK</span></a></li><li class="level-3"><a class="final" href="/sdk/python/"><span>Python SDK</span></a></li><li class="level-3"><a class="final" href="/sdk/ruby/"><span>Ruby SDK</span></a></li><li class="level-3"><a class="final" href="/sdk/community/"><span>Community Powered SDKs</span></a></li></ul></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Deploying an Engine</span></a><ul><li class="level-2"><a class="final" href="/deploy/"><span>Deploying as a Web Service</span></a>< /li><li class="level-2"><a class="final" href="/cli/#engine-commands"><span>Engine Command-line Interface</span></a></li><li class="level-2"><a class="final" href="/deploy/monitoring/"><span>Monitoring Engine</span></a></li><li class="level-2"><a class="final" href="/deploy/engineparams/"><span>Setting Engine Parameters</span></a></li><li class="level-2"><a class="final" href="/deploy/enginevariants/"><span>Deploying Multiple Engine Variants</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Customizing an Engine</span></a><ul><li class="level-2"><a class="final" href="/customize/"><span>Learning DASE</span></a></li><li class="level-2"><a class="final" href="/customize/dase/"><span>Implement DASE</span></a></li><li class="level-2"><a class="final" href="/customize/troubleshooting/"><span>Troubleshooting Engine Development</span></a></li><li class="level-2"><a class="final" href="/api/current/#package"><span>Engine Scala APIs</span></a></li></ul></li><l i class="level-1"><a class="expandible" href="#"><span>Collecting and Analyzing Data</span></a><ul><li class="level-2"><a class="final" href="/datacollection/"><span>Event Server Overview</span></a></li><li class="level-2"><a class="final" href="/cli/#event-server-commands"><span>Event Server Command-line Interface</span></a></li><li class="level-2"><a class="final" href="/datacollection/eventapi/"><span>Collecting Data with REST/SDKs</span></a></li><li class="level-2"><a class="final" href="/datacollection/eventmodel/"><span>Events Modeling</span></a></li><li class="level-2"><a class="final" href="/datacollection/webhooks/"><span>Unifying Multichannel Data with Webhooks</span></a></li><li class="level-2"><a class="final" href="/datacollection/channel/"><span>Channel</span></a></li><li class="level-2"><a class="final" href="/datacollection/batchimport/"><span>Importing Data in Batch</span></a></li><li class="level-2"><a class="final" href="/datacollection/analytics/"><span>Using Ana lytics Tools</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Choosing an Algorithm(s)</span></a><ul><li class="level-2"><a class="final" href="/algorithm/"><span>Built-in Algorithm Libraries</span></a></li><li class="level-2"><a class="final" href="/algorithm/switch/"><span>Switching to Another Algorithm</span></a></li><li class="level-2"><a class="final" href="/algorithm/multiple/"><span>Combining Multiple Algorithms</span></a></li><li class="level-2"><a class="final" href="/algorithm/custom/"><span>Adding Your Own Algorithms</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>ML Tuning and Evaluation</span></a><ul><li class="level-2"><a class="final" href="/evaluation/"><span>Overview</span></a></li><li class="level-2"><a class="final" href="/evaluation/paramtuning/"><span>Hyperparameter Tuning</span></a></li><li class="level-2"><a class="final" href="/evaluation/evaluationdashboard/"><span>Evaluation Dashboard</span> </a></li><li class="level-2"><a class="final" href="/evaluation/metricchoose/"><span>Choosing Evaluation Metrics</span></a></li><li class="level-2"><a class="final" href="/evaluation/metricbuild/"><span>Building Evaluation Metrics</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>System Architecture</span></a><ul><li class="level-2"><a class="final" href="/system/"><span>Architecture Overview</span></a></li><li class="level-2"><a class="final" href="/system/anotherdatastore/"><span>Using Another Data Store</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Engine Template Gallery</span></a><ul><li class="level-2"><a class="final" href="/gallery/template-gallery/"><span>Browse</span></a></li><li class="level-2"><a class="final" href="/community/submit-template/"><span>Submit your Engine as a Template</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Demo Tutorials</span></a><ul><li class="l evel-2"><a class="final" href="/demo/tapster/"><span>Comics Recommendation Demo</span></a></li><li class="level-2"><a class="final" href="/demo/community/"><span>Community Contributed Demo</span></a></li><li class="level-2"><a class="final" href="/demo/textclassification/"><span>Text Classification Engine Tutorial</span></a></li></ul></li><li class="level-1"><a class="expandible" href="/community/"><span>Getting Involved</span></a><ul><li class="level-2"><a class="final" href="/community/contribute-code/"><span>Contribute Code</span></a></li><li class="level-2"><a class="final" href="/community/contribute-documentation/"><span>Contribute Documentation</span></a></li><li class="level-2"><a class="final" href="/community/contribute-sdk/"><span>Contribute a SDK</span></a></li><li class="level-2"><a class="final" href="/community/contribute-webhook/"><span>Contribute a Webhook</span></a></li><li class="level-2"><a class="final" href="/community/projects/"><span>Community Projects</span> </a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Getting Help</span></a><ul><li class="level-2"><a class="final" href="/resources/faq/"><span>FAQs</span></a></li><li class="level-2"><a class="final" href="/support/"><span>Support</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#"><span>Resources</span></a><ul><li class="level-2"><a class="final" href="/resources/intellij/"><span>Developing Engines with IntelliJ IDEA</span></a></li><li class="level-2"><a class="final" href="/resources/upgrade/"><span>Upgrade Instructions</span></a></li><li class="level-2"><a class="final" href="/resources/glossary/"><span>Glossary</span></a></li></ul></li></ul></nav></div><div class="col-md-9 col-sm-12"><div class="content-header hidden-md hidden-lg"><div id="page-title"><h1>Filter Recommended Items by Blacklist in Query (Recommendation)</h1></div></div><div id="table-of-content-wrapper"><h5>On this page</h5><aside id="table-of-contents"><ul> <li> < a href="#add-query-parameter">Add Query Parameter</a> </li> <li> <a href="#filter-the-data">Filter the Data</a> </li> <li> <a href="#put-it-all-together">Put It All Together</a> </li> <li> <a href="#test-the-result">Test the Result</a> </li> </ul> </aside><hr/><a id="edit-page-link" href="https://github.com/apache/incubator-predictionio/tree/livedoc/docs/manual/source/templates/recommendation/blacklist-items.html.md";><img src="/images/icons/edit-pencil-d6c1bb3d.png"/>Edit this page</a></div><div class="content-header hidden-sm hidden-xs"><div id="page-title"><h1>Filter Recommended Items by Blacklist in Query (Recommendation)</h1></div></div><div class="content"><p>Let's say you want to supply a backList for each query to exclude some items from recommendation (For example, in the browsing session, the user just added some items to shopping cart, or you have a list of items you want to filter out, you may want to supply blackList in Query). This how-to will demonstrate how you ca n do it.</p><p>Note that you may also use <a href="/gallery/template-gallery#recommender-systems">E-Commerce Recommendation Template</a> which supports this feature by default.</p><p>If you are looking for filtering out items based on the specific user-to-item events logged by EventServer (eg. filter all items which the user has "buy" events on), you can use the <a href="/gallery/template-gallery#recommender-systems">E-Commerce Recommendation Template</a>. Please refer to the algorithm parameters "unseenOnly" and "seenEvents" of the E-Commerce Recommenation Template.</p><h2 id='add-query-parameter' class='header-anchors'>Add Query Parameter</h2><p>First of all we need to specify query parameter to send items ids that the user has already seen. Lets modify <code>case class Query</code> in MyRecommendation/src/main/scala/<strong><em>Engine.scala</em></strong>:</p><div class="highlight scala"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl " style="text-align: right"><pre class="lineno">1 +2 +3 +4 +5</pre></td><td class="code"><pre><span class="k">case</span> <span class="k">class</span> <span class="nc">Query</span><span class="o">(</span> + <span class="n">user</span><span class="k">:</span> <span class="kt">String</span><span class="o">,</span> + <span class="n">num</span><span class="k">:</span> <span class="kt">Int</span><span class="o">,</span> + <span class="n">blackList</span><span class="k">:</span> <span class="kt">Set</span><span class="o">[</span><span class="kt">String</span><span class="o">]</span> <span class="c1">// NOTE: line added +</span><span class="o">)</span> <span class="k">extends</span> <span class="nc">Serializable</span> +</pre></td></tr></tbody></table> </div> <h2 id='filter-the-data' class='header-anchors'>Filter the Data</h2><p>Then we need to change the code that computes recommendation score to filter out the seen items. Lets modify class MyRecommendation/src/main/scala/<strong><em>ALSModel.scala</em></strong>. Just add the following two methods to that class.</p><div class="highlight scala"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20</pre></td><td class="code"><pre> <span class="k">def</span> <span class="n">recommendProductsWithFilter</span><span class="o">(</span> + <span class="n">user</span><span class="k">:</span> <span class="kt">Int</span><span class="o">,</span> + <span class="n">num</span><span class="k">:</span> <span class="kt">Int</span><span class="o">,</span> + <span class="n">productIdFilter</span><span class="k">:</span> <span class="kt">Set</span><span class="o">[</span><span class="kt">Int</span><span class="o">])</span> <span class="k">=</span> <span class="o">{</span> + <span class="k">val</span> <span class="n">filteredProductFeatures</span> <span class="k">=</span> <span class="n">productFeatures</span> + <span class="o">.</span><span class="n">filter</span><span class="o">(</span><span class="n">features</span> <span class="k">=></span> <span class="o">!</span><span class="n">productIdFilter</span><span class="o">.</span><span class="n">contains</span><span class="o">(</span><span class="n">features</span><span class="o">.</span><span class="n">_1</span><span class="o">))</span> <span class="c1">// (*) +</span> <span class="n">recommend</span><span class="o">(</span><span class="n">userFeatures</span><span class="o">.</span><span class="n">lookup</span><span class="o">(</span><span class="n">user</span><span class="o">).</span><span class="n">head</span><span class="o">,</span> <span class="n">filteredProductFeatures</span><span class="o">,</span> <span class="n">num</span><span class="o">)</span> + <span class="o">.</span><span class="n">map</span><span class="o">(</span><span class="n">t</span> <span class="k">=></span> <span class="nc">Rating</span><span class="o">(</span><span class="n">user</span><span class="o">,</span> <span class="n">t</span><span class="o">.</span><span class="n">_1</span><span class="o">,</span> <span class="n">t</span><span class="o">.</span><span class="n">_2</span><span class="o">))</span> + <span class="o">}</span> + + <span class="k">private</span> <span class="k">def</span> <span class="n">recommend</span><span class="o">(</span> + <span class="n">recommendToFeatures</span><span class="k">:</span> <span class="kt">Array</span><span class="o">[</span><span class="kt">Double</span><span class="o">],</span> + <span class="n">recommendableFeatures</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[(</span><span class="kt">Int</span>, <span class="kt">Array</span><span class="o">[</span><span class="kt">Double</span><span class="o">])],</span> + <span class="n">num</span><span class="k">:</span> <span class="kt">Int</span><span class="o">)</span><span class="k">:</span> <span class="kt">Array</span><span class="o">[(</span><span class="kt">Int</span>, <span class="kt">Double</span><span class="o">)]</span> <span class="k">=</span> <span class="o">{</span> + <span class="k">val</span> <span class="n">recommendToVector</span> <span class="k">=</span> <span class="k">new</span> <span class="nc">DoubleMatrix</span><span class="o">(</span><span class="n">recommendToFeatures</span><span class="o">)</span> + <span class="k">val</span> <span class="n">scored</span> <span class="k">=</span> <span class="n">recommendableFeatures</span><span class="o">.</span><span class="n">map</span> <span class="o">{</span> <span class="k">case</span> <span class="o">(</span><span class="n">id</span><span class="o">,</span><span class="n">features</span><span class="o">)</span> <span class="k">=></span> + <span class="o">(</span><span class="n">id</span><span class="o">,</span> <span class="n">recommendToVector</span><span class="o">.</span><span class="n">dot</span><span class="o">(</span><span class="k">new</span> <span class="nc">DoubleMatrix</span><span class="o">(</span><span class="n">features</span><span class="o">)))</span> + <span class="o">}</span> + <span class="n">scored</span><span class="o">.</span><span class="n">top</span><span class="o">(</span><span class="n">num</span><span class="o">)(</span><span class="nc">Ordering</span><span class="o">.</span><span class="n">by</span><span class="o">(</span><span class="k">_</span><span class="o">.</span><span class="n">_2</span><span class="o">))</span> + <span class="o">}</span> +</pre></td></tr></tbody></table> </div> <p>Also itâs required to add import of the <code>org.jblas.DoubleMatrix</code> class. Please make attention that method <code>recommend</code> is the copy of method <code>org.apache.spark.mllib.recommendation.MatrixFactorizationModel#recommend</code>. We can't reuse this because itâs private. Method <code>recommendProductsWithFilter</code> is the almost full copy of <code>org.apache.spark.mllib.recommendation.MatrixFactorizationModel#recommendProduct</code> method. The difference only is the line with commentary â(*)â where we apply filtering.</p><h2 id='put-it-all-together' class='header-anchors'>Put It All Together</h2><p>Next we need to invoke our new method with filtering when we query recommendations. Lets modify method <code>predict</code> in MyRecommendation/src/main/scala/<strong><em>ALSAlgorithm.scala</em></strong>:</p><div class="highlight scala"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="tex t-align: right"><pre class="lineno">1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17</pre></td><td class="code"><pre> <span class="k">def</span> <span class="n">predict</span><span class="o">(</span><span class="n">model</span><span class="k">:</span> <span class="kt">ALSModel</span><span class="o">,</span> <span class="n">query</span><span class="k">:</span> <span class="kt">Query</span><span class="o">)</span><span class="k">:</span> <span class="kt">PredictedResult</span> <span class="o">=</span> <span class="o">{</span> + <span class="c1">// Convert String ID to Int index for Mllib +</span> <span class="n">model</span><span class="o">.</span><span class="n">userStringIntMap</span><span class="o">.</span><span class="n">get</span><span class="o">(</span><span class="n">query</span><span class="o">.</span><span class="n">user</span><span class="o">).</span><span class="n">map</span> <span class="o">{</span> <span class="n">userInt</span> <span class="k">=></span> + <span class="c1">// create inverse view of itemStringIntMap +</span> <span class="k">val</span> <span class="n">itemIntStringMap</span> <span class="k">=</span> <span class="n">model</span><span class="o">.</span><span class="n">itemStringIntMap</span><span class="o">.</span><span class="n">inverse</span> + <span class="c1">// recommendProducts() returns Array[MLlibRating], which uses item Int +</span> <span class="c1">// index. Convert it to String ID for returning PredictedResult +</span> <span class="k">val</span> <span class="n">blackListedIds</span> <span class="k">=</span> <span class="n">query</span><span class="o">.</span><span class="n">blackList</span><span class="o">.</span><span class="n">flatMap</span><span class="o">(</span><span class="n">model</span><span class="o">.</span><span class="n">itemStringIntMap</span><span class="o">.</span><span class="n">get</span><span class="o">)</span> <span class="c1">// NOTE: line added +</span> <span class="k">val</span> <span class="n">itemScores</span> <span class="k">=</span> <span class="n">model</span> + <span class="o">.</span><span class="n">recommendProductsWithFilter</span><span class="o">(</span><span class="n">userInt</span><span class="o">,</span> <span class="n">query</span><span class="o">.</span><span class="n">num</span><span class="o">,</span> <span class="n">blackListedIds</span><span class="o">)</span> <span class="c1">// NOTE: line modified +</span> <span class="o">.</span><span class="n">map</span><span class="o">(</span><span class="n">r</span> <span class="k">=></span> <span class="nc">ItemScore</span><span class="o">(</span><span class="n">itemIntStringMap</span><span class="o">(</span><span class="n">r</span><span class="o">.</span><span class="n">product</span><span class="o">),</span> <span class="n">r</span><span class="o">.</span><span class="n">rating</span><span class="o">))</span> + <span class="k">new</span> <span class="nc">PredictedResult</span><span class="o">(</span><span class="n">itemScores</span><span class="o">)</span> + <span class="o">}.</span><span class="n">getOrElse</span><span class="o">{</span> + <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="o">(</span><span class="n">s</span><span class="s">"No prediction for unknown user ${query.user}."</span><span class="o">)</span> + <span class="k">new</span> <span class="nc">PredictedResult</span><span class="o">(</span><span class="nc">Array</span><span class="o">.</span><span class="n">empty</span><span class="o">)</span> + <span class="o">}</span> + <span class="o">}</span> +</pre></td></tr></tbody></table> </div> <h2 id='test-the-result' class='header-anchors'>Test the Result</h2><p>Then we can build/train/deploy the engine and test the result:</p><p>The query</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4</pre></td><td class="code"><pre>curl <span class="se">\</span> +-H <span class="s2">"Content-Type: application/json"</span> <span class="se">\</span> +-d <span class="s1">'{ "user": "1", "num": 4 }'</span> <span class="se">\</span> +http://localhost:8000/queries.json +</pre></td></tr></tbody></table> </div> <p>will return the result</p><div class="highlight json"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15</pre></td><td class="code"><pre><span class="p">{</span><span class="w"> + </span><span class="s2">"itemScores"</span><span class="p">:</span><span class="w"> </span><span class="p">[{</span><span class="w"> + </span><span class="s2">"item"</span><span class="p">:</span><span class="w"> </span><span class="s2">"32"</span><span class="p">,</span><span class="w"> + </span><span class="s2">"score"</span><span class="p">:</span><span class="w"> </span><span class="mf">13.405593705856901</span><span class="w"> + </span><span class="p">},</span><span class="w"> </span><span class="p">{</span><span class="w"> + </span><span class="s2">"item"</span><span class="p">:</span><span class="w"> </span><span class="s2">"90"</span><span class="p">,</span><span class="w"> + </span><span class="s2">"score"</span><span class="p">:</span><span class="w"> </span><span class="mf">10.980439687813178</span><span class="w"> + </span><span class="p">},</span><span class="w"> </span><span class="p">{</span><span class="w"> + </span><span class="s2">"item"</span><span class="p">:</span><span class="w"> </span><span class="s2">"75"</span><span class="p">,</span><span class="w"> + </span><span class="s2">"score"</span><span class="p">:</span><span class="w"> </span><span class="mf">10.748973860065737</span><span class="w"> + </span><span class="p">},</span><span class="w"> </span><span class="p">{</span><span class="w"> + </span><span class="s2">"item"</span><span class="p">:</span><span class="w"> </span><span class="s2">"1"</span><span class="p">,</span><span class="w"> + </span><span class="s2">"score"</span><span class="p">:</span><span class="w"> </span><span class="mf">9.769636099226231</span><span class="w"> + </span><span class="p">}]</span><span class="w"> +</span><span class="p">}</span><span class="w"> +</span></pre></td></tr></tbody></table> </div> <p>Lets say that the user has seen the <code>32</code> item.</p><div class="highlight shell"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4</pre></td><td class="code"><pre>curl <span class="se">\</span> +-H <span class="s2">"Content-Type: application/json"</span> <span class="se">\</span> +-d <span class="s1">'{ "user": "1", "num": 4, "blackList": ["32"] }'</span> <span class="se">\</span> +http://localhost:8000/queries.json +</pre></td></tr></tbody></table> </div> <p>will return the result</p><div class="highlight json"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15</pre></td><td class="code"><pre><span class="p">{</span><span class="w"> + </span><span class="s2">"itemScores"</span><span class="p">:</span><span class="w"> </span><span class="p">[{</span><span class="w"> + </span><span class="s2">"item"</span><span class="p">:</span><span class="w"> </span><span class="s2">"90"</span><span class="p">,</span><span class="w"> + </span><span class="s2">"score"</span><span class="p">:</span><span class="w"> </span><span class="mf">10.980439687813178</span><span class="w"> + </span><span class="p">},</span><span class="w"> </span><span class="p">{</span><span class="w"> + </span><span class="s2">"item"</span><span class="p">:</span><span class="w"> </span><span class="s2">"75"</span><span class="p">,</span><span class="w"> + </span><span class="s2">"score"</span><span class="p">:</span><span class="w"> </span><span class="mf">10.748973860065737</span><span class="w"> + </span><span class="p">},</span><span class="w"> </span><span class="p">{</span><span class="w"> + </span><span class="s2">"item"</span><span class="p">:</span><span class="w"> </span><span class="s2">"1"</span><span class="p">,</span><span class="w"> + </span><span class="s2">"score"</span><span class="p">:</span><span class="w"> </span><span class="mf">9.769636099226231</span><span class="w"> + </span><span class="p">},</span><span class="w"> </span><span class="p">{</span><span class="w"> + </span><span class="s2">"item"</span><span class="p">:</span><span class="w"> </span><span class="s2">"49"</span><span class="p">,</span><span class="w"> + </span><span class="s2">"score"</span><span class="p">:</span><span class="w"> </span><span class="mf">8.653951817512265</span><span class="w"> + </span><span class="p">}]</span><span class="w"> +</span><span class="p">}</span><span class="w"> +</span></pre></td></tr></tbody></table> </div> <p>without item <code>32</code>.</p></div></div></div></div><footer><div class="container"><div class="seperator"></div><div class="row"><div class="col-md-6 col-xs-6 footer-link-column"><div class="footer-link-column-row"><h4>Community</h4><ul><li><a href="//docs.prediction.io/install/" target="blank">Download</a></li><li><a href="//docs.prediction.io/" target="blank">Docs</a></li><li><a href="//github.com/apache/incubator-predictionio" target="blank">GitHub</a></li><li><a href="mailto:user-subscr...@predictionio.incubator.apache.org"; target="blank">Subscribe to User Mailing List</a></li><li><a href="//stackoverflow.com/questions/tagged/predictionio" target="blank">Stackoverflow</a></li></ul></div></div><div class="col-md-6 col-xs-6 footer-link-column"><div class="footer-link-column-row"><h4>Contribute</h4><ul><li><a href="//predictionio.incubator.apache.org/community/contribute-code/" target="blank">Contribute</a></li><li><a href="//g ithub.com/apache/incubator-predictionio" target="blank">Source Code</a></li><li><a href="//issues.apache.org/jira/browse/PIO" target="blank">Bug Tracker</a></li><li><a href="mailto:dev-subscr...@predictionio.incubator.apache.org"; target="blank">Subscribe to Development Mailing List</a></li></ul></div></div></div></div><div id="footer-bottom"><div class="container"><div class="row"><div class="col-md-12"><div id="footer-logo-wrapper"><img alt="PredictionIO" src="/images/logos/logo-white-d1e9c6e6.png"/></div><div id="social-icons-wrapper"><a class="github-button" href="https://github.com/apache/incubator-predictionio"; data-style="mega" data-count-href="/apache/incubator-predictionio/stargazers" data-count-api="/repos/apache/incubator-predictionio#stargazers_count" data-count-aria-label="# stargazers on GitHub" aria-label="Star apache/incubator-predictionio on GitHub">Star</a> <a class="github-button" href="https://github.com/apache/incubator-predictionio/fork"; data-icon="octicon-git-b ranch" data-style="mega" data-count-href="/apache/incubator-predictionio/network" data-count-api="/repos/apache/incubator-predictionio#forks_count" data-count-aria-label="# forks on GitHub" aria-label="Fork apache/incubator-predictionio on GitHub">Fork</a> <script id="github-bjs" async="" defer="" src="https://buttons.github.io/buttons.js";></script><a href="//www.facebook.com/predictionio" target="blank"><img alt="PredictionIO on Twitter" src="/images/icons/twitter-ea9dc152.png"/></a> <a href="//twitter.com/predictionio" target="blank"><img alt="PredictionIO on Facebook" src="/images/icons/facebook-5c57939c.png"/></a> </div></div></div></div></div></footer></div><script>(function(w,d,t,u,n,s,e){w['SwiftypeObject']=n;w[n]=w[n]||function(){ +(w[n].q=w[n].q||[]).push(arguments);};s=d.createElement(t); +e=d.getElementsByTagName(t)[0];s.async=1;s.src=u;e.parentNode.insertBefore(s,e); +})(window,document,'script','//s.swiftypecdn.com/install/v1/st.js','_st'); + +_st('install','HaUfpXXV87xoB_zzCQ45');</script><script src="/javascripts/application-280db181.js"></script></body></html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-predictionio-site/blob/25938169/templates/recommendation/blacklist-items/index.html.gz ---------------------------------------------------------------------- diff --git a/templates/recommendation/blacklist-items/index.html.gz b/templates/recommendation/blacklist-items/index.html.gz new file mode 100644 index 0000000..75cdcff Binary files /dev/null and b/templates/recommendation/blacklist-items/index.html.gz differ
