http://git-wip-us.apache.org/repos/asf/predictionio-site/blob/1edaec49/evaluation/paramtuning/index.html
----------------------------------------------------------------------
diff --git a/evaluation/paramtuning/index.html
b/evaluation/paramtuning/index.html
index cbc0b5b..5c5b0d1 100644
--- a/evaluation/paramtuning/index.html
+++ b/evaluation/paramtuning/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html><head><title>Hyperparameter Tuning</title><meta
charset="utf-8"/><meta content="IE=edge,chrome=1"
http-equiv="X-UA-Compatible"/><meta name="viewport"
content="width=device-width, initial-scale=1.0"/><meta class="swiftype"
name="title" data-type="string" content="Hyperparameter Tuning"/><link
rel="canonical"
href="https://predictionio.apache.org/evaluation/paramtuning/"/><link
href="/images/favicon/normal-b330020a.png" rel="shortcut icon"/><link
href="/images/favicon/apple-c0febcf2.png" rel="apple-touch-icon"/><link
href="//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800"
rel="stylesheet"/><link
href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css"
rel="stylesheet"/><link href="/stylesheets/application-eccfc6cb.css"
rel="stylesheet" type="text/css"/><script
src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.2/html5shiv.min.js"></script><script
src="//cdn.mathjax.org/mathjax/la
test/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script><script
src="//use.typekit.net/pqo0itb.js"></script><script>try{Typekit.load({ async:
true });}catch(e){}</script></head><body><div id="global"><header><div
class="container" id="header-wrapper"><div class="row"><div
class="col-sm-12"><div id="logo-wrapper"><span id="drawer-toggle"></span><a
href="#"></a><a href="http://predictionio.apache.org/";><img alt="Apache
PredictionIO" id="logo"
src="/images/logos/logo-ee2b9bb3.png"/></a><span>â¢</span></div><div
id="menu-wrapper"><div id="pill-wrapper"><a class="pill left"
href="/gallery/template-gallery">TEMPLATES</a> <a class="pill right"
href="//github.com/apache/incubator-predictionio/">OPEN
SOURCE</a></div></div><img class="mobile-search-bar-toggler hidden-md
hidden-lg"
src="/images/icons/search-glass-704bd4ff.png"/></div></div></div></header><div
id="search-bar-row-wrapper"><div class="container-fluid"
id="search-bar-row"><div class="row"><div class="col-md-9 col-sm-11 col-xs-11">
<div class="hidden-md hidden-lg"
id="mobile-page-heading-wrapper"><p>PredictionIO Docs</p><h4>Hyperparameter
Tuning</h4></div><h4 class="hidden-sm hidden-xs">PredictionIO
Docs</h4></div><div class="col-md-3 col-sm-1 col-xs-1 hidden-md hidden-lg"><img
id="left-menu-indicator"
src="/images/icons/down-arrow-dfe9f7fe.png"/></div><div class="col-md-3
col-sm-12 col-xs-12 swiftype-wrapper"><div class="swiftype"><form
class="search-form"><img class="search-box-toggler hidden-xs hidden-sm"
src="/images/icons/search-glass-704bd4ff.png"/><div class="search-box"><img
src="/images/icons/search-glass-704bd4ff.png"/><input type="text"
id="st-search-input" class="st-search-input" placeholder="Search
Doc..."/></div><img class="swiftype-row-hider hidden-md hidden-lg"
src="/images/icons/drawer-toggle-active-fcbef12a.png"/></form></div></div><div
class="mobile-left-menu-toggler hidden-md
hidden-lg"></div></div></div></div><div id="page" class="container-fluid"><div
class="row"><div id="left-menu-wrappe
r" class="col-md-3"><nav id="nav-main"><ul><li class="level-1"><a
class="expandible" href="/"><span>Apache PredictionIO®
Documentation</span></a><ul><li class="level-2"><a class="final"
href="/"><span>Welcome to Apache PredictionIO®</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Getting
Started</span></a><ul><li class="level-2"><a class="final"
href="/start/"><span>A Quick Intro</span></a></li><li class="level-2"><a
class="final" href="/install/"><span>Installing Apache
PredictionIO</span></a></li><li class="level-2"><a class="final"
href="/start/download/"><span>Downloading an Engine Template</span></a></li><li
class="level-2"><a class="final" href="/start/deploy/"><span>Deploying Your
First Engine</span></a></li><li class="level-2"><a class="final"
href="/start/customize/"><span>Customizing the
Engine</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Integrating with Your App</span></a><ul><li class="level-2"><a
class="final" href="/appintegration/"><span>App Integration
Overview</span></a></li><li class="level-2"><a class="expandible"
href="/sdk/"><span>List of SDKs</span></a><ul><li class="level-3"><a
class="final" href="/sdk/java/"><span>Java & Android SDK</span></a></li><li
class="level-3"><a class="final" href="/sdk/php/"><span>PHP
SDK</span></a></li><li class="level-3"><a class="final"
href="/sdk/python/"><span>Python SDK</span></a></li><li class="level-3"><a
class="final" href="/sdk/ruby/"><span>Ruby SDK</span></a></li><li
class="level-3"><a class="final" href="/sdk/community/"><span>Community Powered
SDKs</span></a></li></ul></li></ul></li><li class="level-1"><a
class="expandible" href="#"><span>Deploying an Engine</span></a><ul><li
class="level-2"><a class="final" href="/deploy/"><span>Deploying as a Web
Service</span></a></li><li class="level-2"><a class="final"
href="/batchpredict/"><span>Batch Predictions</span></a></li><li
class="level-2"><a class="final" href="/deploy/monitori
ng/"><span>Monitoring Engine</span></a></li><li class="level-2"><a
class="final" href="/deploy/engineparams/"><span>Setting Engine
Parameters</span></a></li><li class="level-2"><a class="final"
href="/deploy/enginevariants/"><span>Deploying Multiple Engine
Variants</span></a></li><li class="level-2"><a class="final"
href="/deploy/plugin/"><span>Engine Server Plugin</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Customizing an
Engine</span></a><ul><li class="level-2"><a class="final"
href="/customize/"><span>Learning DASE</span></a></li><li class="level-2"><a
class="final" href="/customize/dase/"><span>Implement DASE</span></a></li><li
class="level-2"><a class="final"
href="/customize/troubleshooting/"><span>Troubleshooting Engine
Development</span></a></li><li class="level-2"><a class="final"
href="/api/current/#package"><span>Engine Scala
APIs</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Collecting and Analyzin
g Data</span></a><ul><li class="level-2"><a class="final"
href="/datacollection/"><span>Event Server Overview</span></a></li><li
class="level-2"><a class="final"
href="/datacollection/eventapi/"><span>Collecting Data with
REST/SDKs</span></a></li><li class="level-2"><a class="final"
href="/datacollection/eventmodel/"><span>Events Modeling</span></a></li><li
class="level-2"><a class="final"
href="/datacollection/webhooks/"><span>Unifying Multichannel Data with
Webhooks</span></a></li><li class="level-2"><a class="final"
href="/datacollection/channel/"><span>Channel</span></a></li><li
class="level-2"><a class="final"
href="/datacollection/batchimport/"><span>Importing Data in
Batch</span></a></li><li class="level-2"><a class="final"
href="/datacollection/analytics/"><span>Using Analytics
Tools</span></a></li><li class="level-2"><a class="final"
href="/datacollection/plugin/"><span>Event Server
Plugin</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Cho
osing an Algorithm(s)</span></a><ul><li class="level-2"><a class="final"
href="/algorithm/"><span>Built-in Algorithm Libraries</span></a></li><li
class="level-2"><a class="final" href="/algorithm/switch/"><span>Switching to
Another Algorithm</span></a></li><li class="level-2"><a class="final"
href="/algorithm/multiple/"><span>Combining Multiple
Algorithms</span></a></li><li class="level-2"><a class="final"
href="/algorithm/custom/"><span>Adding Your Own
Algorithms</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>ML Tuning and Evaluation</span></a><ul><li class="level-2"><a
class="final" href="/evaluation/"><span>Overview</span></a></li><li
class="level-2"><a class="final active"
href="/evaluation/paramtuning/"><span>Hyperparameter Tuning</span></a></li><li
class="level-2"><a class="final"
href="/evaluation/evaluationdashboard/"><span>Evaluation
Dashboard</span></a></li><li class="level-2"><a class="final"
href="/evaluation/metricchoose/"><span>Choosi
ng Evaluation Metrics</span></a></li><li class="level-2"><a class="final"
href="/evaluation/metricbuild/"><span>Building Evaluation
Metrics</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>System Architecture</span></a><ul><li class="level-2"><a
class="final" href="/system/"><span>Architecture Overview</span></a></li><li
class="level-2"><a class="final" href="/system/anotherdatastore/"><span>Using
Another Data Store</span></a></li></ul></li><li class="level-1"><a
class="expandible" href="#"><span>PredictionIO® Official
Templates</span></a><ul><li class="level-2"><a class="final"
href="/templates/"><span>Intro</span></a></li><li class="level-2"><a
class="expandible" href="#"><span>Recommendation</span></a><ul><li
class="level-3"><a class="final"
href="/templates/recommendation/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/dase/"><span>DASE</span></a></li><li
class="level-3"><a class
="final" href="/templates/recommendation/evaluation/"><span>Evaluation
Explained</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/recommendation/reading-custom-events/"><span>Read Custom
Events</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/customize-data-prep/"><span>Customize Data
Preparator</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/customize-serving/"><span>Customize
Serving</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/training-with-implicit-preference/"><span>Train
with Implicit Preference</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/blacklist-items/"><span>Filter Recommended
Items by Blacklist in Query</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/batch-evaluator
/"><span>Batch Persistable Evaluator</span></a></li></ul></li><li
class="level-2"><a class="expandible" href="#"><span>E-Commerce
Recommendation</span></a><ul><li class="level-3"><a class="final"
href="/templates/ecommercerecommendation/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/ecommercerecommendation/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/ecommercerecommendation/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/ecommercerecommendation/train-with-rate-event/"><span>Train
with Rate Event</span></a></li><li class="level-3"><a class="final"
href="/templates/ecommercerecommendation/adjust-score/"><span>Adjust
Score</span></a></li></ul></li><li class="level-2"><a class="expandible"
href="#"><span>Similar Product</span></a><ul><li class="level-3"><a
class="final" href="/templates/similarproduct/quickstart/"><span>Quick
Start</span></a></li><li
class="level-3"><a class="final"
href="/templates/similarproduct/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/similarproduct/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/similarproduct/multi-events-multi-algos/"><span>Multiple
Events and Multiple Algorithms</span></a></li><li class="level-3"><a
class="final"
href="/templates/similarproduct/return-item-properties/"><span>Returns Item
Properties</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/train-with-rate-event/"><span>Train with Rate
Event</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/rid-user-set-event/"><span>Get Rid of Events
for Users</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/recommended-user/"><span>Recommend
Users</span></a></li></ul></li><li class="level-2"><a class="expandible"
href="#"><span>Classification</span></a><ul><
li class="level-3"><a class="final"
href="/templates/classification/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/classification/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/classification/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/classification/add-algorithm/"><span>Use Alternative
Algorithm</span></a></li><li class="level-3"><a class="final"
href="/templates/classification/reading-custom-properties/"><span>Read Custom
Properties</span></a></li></ul></li></ul></li><li class="level-1"><a
class="expandible" href="#"><span>Engine Template Gallery</span></a><ul><li
class="level-2"><a class="final"
href="/gallery/template-gallery/"><span>Browse</span></a></li><li
class="level-2"><a class="final"
href="/community/submit-template/"><span>Submit your Engine as a
Template</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><
span>Demo Tutorials</span></a><ul><li class="level-2"><a class="final"
href="/demo/tapster/"><span>Comics Recommendation Demo</span></a></li><li
class="level-2"><a class="final" href="/demo/community/"><span>Community
Contributed Demo</span></a></li><li class="level-2"><a class="final"
href="/demo/textclassification/"><span>Text Classification Engine
Tutorial</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="/community/"><span>Getting Involved</span></a><ul><li class="level-2"><a
class="final" href="/community/contribute-code/"><span>Contribute
Code</span></a></li><li class="level-2"><a class="final"
href="/community/contribute-documentation/"><span>Contribute
Documentation</span></a></li><li class="level-2"><a class="final"
href="/community/contribute-sdk/"><span>Contribute a SDK</span></a></li><li
class="level-2"><a class="final"
href="/community/contribute-webhook/"><span>Contribute a
Webhook</span></a></li><li class="level-2"><a class="final" href="/commun
ity/projects/"><span>Community Projects</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Getting
Help</span></a><ul><li class="level-2"><a class="final"
href="/resources/faq/"><span>FAQs</span></a></li><li class="level-2"><a
class="final" href="/support/"><span>Support</span></a></li></ul></li><li
class="level-1"><a class="expandible"
href="#"><span>Resources</span></a><ul><li class="level-2"><a class="final"
href="/cli/"><span>Command-line Interface</span></a></li><li class="level-2"><a
class="final" href="/resources/release/"><span>Release
Cadence</span></a></li><li class="level-2"><a class="final"
href="/resources/intellij/"><span>Developing Engines with IntelliJ
IDEA</span></a></li><li class="level-2"><a class="final"
href="/resources/upgrade/"><span>Upgrade Instructions</span></a></li><li
class="level-2"><a class="final"
href="/resources/glossary/"><span>Glossary</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>A
pache Software Foundation</span></a><ul><li class="level-2"><a class="final"
href="https://www.apache.org/";><span>Apache Homepage</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/licenses/";><span>License</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/foundation/sponsorship.html";><span>Sponsorship</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/foundation/thanks.html";><span>Thanks</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/security/";><span>Security</span></a></li></ul></li></ul></nav></div><div
class="col-md-9 col-sm-12"><div class="content-header hidden-md
hidden-lg"><div id="breadcrumbs" class="hidden-sm hidden xs"><ul><li><a
href="#">ML Tuning and Evaluation</a><span
class="spacer">></span></li><li><span class="last">Hyperparameter
Tuning</span></li></ul></div><div id="page-title"><h1>Hyperparameter
Tuning</h1></div></div><div id="table-of-con
tent-wrapper"><h5>On this page</h5><aside id="table-of-contents"><ul> <li> <a
href="#quick-start">Quick Start</a> </li> <li> <a
href="#detailed-explanation">Detailed Explanation</a> </li> <li> <a
href="#the-evaluation-design">The Evaluation Design</a> </li> <li> <a
href="#evaluation-data-generation">Evaluation Data Generation</a> </li> <li> <a
href="#evaluation-metrics">Evaluation Metrics</a> </li> <li> <a
href="#parameters-generation">Parameters Generation</a> </li> <li> <a
href="#running-the-evaluation">Running the Evaluation</a> </li> <li> <a
href="#notes">Notes</a> </li> </ul> </aside><hr/><a id="edit-page-link"
href="https://github.com/apache/incubator-predictionio/tree/livedoc/docs/manual/source/evaluation/paramtuning.html.md";><img
src="/images/icons/edit-pencil-d6c1bb3d.png"/>Edit this page</a></div><div
class="content-header hidden-sm hidden-xs"><div id="breadcrumbs"
class="hidden-sm hidden xs"><ul><li><a href="#">ML Tuning and
Evaluation</a><span class="spacer">></span><
/li><li><span class="last">Hyperparameter Tuning</span></li></ul></div><div
id="page-title"><h1>Hyperparameter Tuning</h1></div></div><div class="content">
<p>A PredictionIO engine is instantiated by a set of parameters. These
parameters define which algorithm is to be used, as well supply the parameters
for the algorithm itself. This naturally raises the question of how to choose
the best set of parameters. The evaluation module streamlines the process of
<em>tuning</em> the engine to the best parameter set and deploys it.</p><h2
id='quick-start' class='header-anchors'>Quick Start</h2><p>We demonstrate the
evaluation with <a href="/templates/classification/quickstart/">the
classification template</a>. The classification template uses a naive bayesian
algorithm that has a smoothing parameter. We evaluate the prediction quality
against different parameter values to find the best parameter values, and then
deploy it.</p><h3 id='edit-the-appid' class='header-anchors'>Edit the AppId</h3
><p>Edit
>MyClassification/src/main/scala/<strong><em>Evaluation.scala</em></strong> to
>specify the <em>appId</em> you used to import the data.</p><div
>class="highlight scala"><table style="border-spacing: 0"><tbody><tr><td
>class="gutter gl" style="text-align: right"><pre class="lineno">1
+<!DOCTYPE html><html><head><title>Hyperparameter Tuning</title><meta
charset="utf-8"/><meta content="IE=edge,chrome=1"
http-equiv="X-UA-Compatible"/><meta name="viewport"
content="width=device-width, initial-scale=1.0"/><meta class="swiftype"
name="title" data-type="string" content="Hyperparameter Tuning"/><link
rel="canonical"
href="https://predictionio.apache.org/evaluation/paramtuning/"/><link
href="/images/favicon/normal-b330020a.png" rel="shortcut icon"/><link
href="/images/favicon/apple-c0febcf2.png" rel="apple-touch-icon"/><link
href="//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800"
rel="stylesheet"/><link
href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css"
rel="stylesheet"/><link href="/stylesheets/application-eccfc6cb.css"
rel="stylesheet" type="text/css"/><script
src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.2/html5shiv.min.js"></script><script
src="//cdn.mathjax.org/mathjax/la
test/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script><script
src="//use.typekit.net/pqo0itb.js"></script><script>try{Typekit.load({ async:
true });}catch(e){}</script></head><body><div id="global"><header><div
class="container" id="header-wrapper"><div class="row"><div
class="col-sm-12"><div id="logo-wrapper"><span id="drawer-toggle"></span><a
href="#"></a><a href="http://predictionio.apache.org/";><img alt="Apache
PredictionIO" id="logo"
src="/images/logos/logo-ee2b9bb3.png"/></a><span>®</span></div><div
id="menu-wrapper"><div id="pill-wrapper"><a class="pill left"
href="/gallery/template-gallery">TEMPLATES</a> <a class="pill right"
href="//github.com/apache/incubator-predictionio/">OPEN
SOURCE</a></div></div><img class="mobile-search-bar-toggler hidden-md
hidden-lg"
src="/images/icons/search-glass-704bd4ff.png"/></div></div></div></header><div
id="search-bar-row-wrapper"><div class="container-fluid"
id="search-bar-row"><div class="row"><div class="col-md-9 col-sm-11 col-xs-11"><
div class="hidden-md hidden-lg"
id="mobile-page-heading-wrapper"><p>PredictionIO Docs</p><h4>Hyperparameter
Tuning</h4></div><h4 class="hidden-sm hidden-xs">PredictionIO
Docs</h4></div><div class="col-md-3 col-sm-1 col-xs-1 hidden-md hidden-lg"><img
id="left-menu-indicator"
src="/images/icons/down-arrow-dfe9f7fe.png"/></div><div class="col-md-3
col-sm-12 col-xs-12 swiftype-wrapper"><div class="swiftype"><form
class="search-form"><img class="search-box-toggler hidden-xs hidden-sm"
src="/images/icons/search-glass-704bd4ff.png"/><div class="search-box"><img
src="/images/icons/search-glass-704bd4ff.png"/><input type="text"
id="st-search-input" class="st-search-input" placeholder="Search
Doc..."/></div><img class="swiftype-row-hider hidden-md hidden-lg"
src="/images/icons/drawer-toggle-active-fcbef12a.png"/></form></div></div><div
class="mobile-left-menu-toggler hidden-md
hidden-lg"></div></div></div></div><div id="page" class="container-fluid"><div
class="row"><div id="left-menu-wrapper
" class="col-md-3"><nav id="nav-main"><ul><li class="level-1"><a
class="expandible" href="/"><span>Apache PredictionIO®
Documentation</span></a><ul><li class="level-2"><a class="final"
href="/"><span>Welcome to Apache PredictionIO®</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Getting
Started</span></a><ul><li class="level-2"><a class="final"
href="/start/"><span>A Quick Intro</span></a></li><li class="level-2"><a
class="final" href="/install/"><span>Installing Apache
PredictionIO</span></a></li><li class="level-2"><a class="final"
href="/start/download/"><span>Downloading an Engine Template</span></a></li><li
class="level-2"><a class="final" href="/start/deploy/"><span>Deploying Your
First Engine</span></a></li><li class="level-2"><a class="final"
href="/start/customize/"><span>Customizing the
Engine</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Integrating with Your App</span></a><ul><li class="level-2"><a c
lass="final" href="/appintegration/"><span>App Integration
Overview</span></a></li><li class="level-2"><a class="expandible"
href="/sdk/"><span>List of SDKs</span></a><ul><li class="level-3"><a
class="final" href="/sdk/java/"><span>Java & Android SDK</span></a></li><li
class="level-3"><a class="final" href="/sdk/php/"><span>PHP
SDK</span></a></li><li class="level-3"><a class="final"
href="/sdk/python/"><span>Python SDK</span></a></li><li class="level-3"><a
class="final" href="/sdk/ruby/"><span>Ruby SDK</span></a></li><li
class="level-3"><a class="final" href="/sdk/community/"><span>Community Powered
SDKs</span></a></li></ul></li></ul></li><li class="level-1"><a
class="expandible" href="#"><span>Deploying an Engine</span></a><ul><li
class="level-2"><a class="final" href="/deploy/"><span>Deploying as a Web
Service</span></a></li><li class="level-2"><a class="final"
href="/batchpredict/"><span>Batch Predictions</span></a></li><li
class="level-2"><a class="final" href="/deploy/monitorin
g/"><span>Monitoring Engine</span></a></li><li class="level-2"><a
class="final" href="/deploy/engineparams/"><span>Setting Engine
Parameters</span></a></li><li class="level-2"><a class="final"
href="/deploy/enginevariants/"><span>Deploying Multiple Engine
Variants</span></a></li><li class="level-2"><a class="final"
href="/deploy/plugin/"><span>Engine Server Plugin</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Customizing an
Engine</span></a><ul><li class="level-2"><a class="final"
href="/customize/"><span>Learning DASE</span></a></li><li class="level-2"><a
class="final" href="/customize/dase/"><span>Implement DASE</span></a></li><li
class="level-2"><a class="final"
href="/customize/troubleshooting/"><span>Troubleshooting Engine
Development</span></a></li><li class="level-2"><a class="final"
href="/api/current/#package"><span>Engine Scala
APIs</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Collecting and Analyzing
Data</span></a><ul><li class="level-2"><a class="final"
href="/datacollection/"><span>Event Server Overview</span></a></li><li
class="level-2"><a class="final"
href="/datacollection/eventapi/"><span>Collecting Data with
REST/SDKs</span></a></li><li class="level-2"><a class="final"
href="/datacollection/eventmodel/"><span>Events Modeling</span></a></li><li
class="level-2"><a class="final"
href="/datacollection/webhooks/"><span>Unifying Multichannel Data with
Webhooks</span></a></li><li class="level-2"><a class="final"
href="/datacollection/channel/"><span>Channel</span></a></li><li
class="level-2"><a class="final"
href="/datacollection/batchimport/"><span>Importing Data in
Batch</span></a></li><li class="level-2"><a class="final"
href="/datacollection/analytics/"><span>Using Analytics
Tools</span></a></li><li class="level-2"><a class="final"
href="/datacollection/plugin/"><span>Event Server
Plugin</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Choo
sing an Algorithm(s)</span></a><ul><li class="level-2"><a class="final"
href="/algorithm/"><span>Built-in Algorithm Libraries</span></a></li><li
class="level-2"><a class="final" href="/algorithm/switch/"><span>Switching to
Another Algorithm</span></a></li><li class="level-2"><a class="final"
href="/algorithm/multiple/"><span>Combining Multiple
Algorithms</span></a></li><li class="level-2"><a class="final"
href="/algorithm/custom/"><span>Adding Your Own
Algorithms</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>ML Tuning and Evaluation</span></a><ul><li class="level-2"><a
class="final" href="/evaluation/"><span>Overview</span></a></li><li
class="level-2"><a class="final active"
href="/evaluation/paramtuning/"><span>Hyperparameter Tuning</span></a></li><li
class="level-2"><a class="final"
href="/evaluation/evaluationdashboard/"><span>Evaluation
Dashboard</span></a></li><li class="level-2"><a class="final"
href="/evaluation/metricchoose/"><span>Choosin
g Evaluation Metrics</span></a></li><li class="level-2"><a class="final"
href="/evaluation/metricbuild/"><span>Building Evaluation
Metrics</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>System Architecture</span></a><ul><li class="level-2"><a
class="final" href="/system/"><span>Architecture Overview</span></a></li><li
class="level-2"><a class="final" href="/system/anotherdatastore/"><span>Using
Another Data Store</span></a></li></ul></li><li class="level-1"><a
class="expandible" href="#"><span>PredictionIO® Official
Templates</span></a><ul><li class="level-2"><a class="final"
href="/templates/"><span>Intro</span></a></li><li class="level-2"><a
class="expandible" href="#"><span>Recommendation</span></a><ul><li
class="level-3"><a class="final"
href="/templates/recommendation/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/dase/"><span>DASE</span></a></li><li
class="level-3"><a class=
"final" href="/templates/recommendation/evaluation/"><span>Evaluation
Explained</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/recommendation/reading-custom-events/"><span>Read Custom
Events</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/customize-data-prep/"><span>Customize Data
Preparator</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/customize-serving/"><span>Customize
Serving</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/training-with-implicit-preference/"><span>Train
with Implicit Preference</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/blacklist-items/"><span>Filter Recommended
Items by Blacklist in Query</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/batch-evaluator/
"><span>Batch Persistable Evaluator</span></a></li></ul></li><li
class="level-2"><a class="expandible" href="#"><span>E-Commerce
Recommendation</span></a><ul><li class="level-3"><a class="final"
href="/templates/ecommercerecommendation/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/ecommercerecommendation/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/ecommercerecommendation/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/ecommercerecommendation/train-with-rate-event/"><span>Train
with Rate Event</span></a></li><li class="level-3"><a class="final"
href="/templates/ecommercerecommendation/adjust-score/"><span>Adjust
Score</span></a></li></ul></li><li class="level-2"><a class="expandible"
href="#"><span>Similar Product</span></a><ul><li class="level-3"><a
class="final" href="/templates/similarproduct/quickstart/"><span>Quick
Start</span></a></li><li c
lass="level-3"><a class="final"
href="/templates/similarproduct/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/similarproduct/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/similarproduct/multi-events-multi-algos/"><span>Multiple
Events and Multiple Algorithms</span></a></li><li class="level-3"><a
class="final"
href="/templates/similarproduct/return-item-properties/"><span>Returns Item
Properties</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/train-with-rate-event/"><span>Train with Rate
Event</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/rid-user-set-event/"><span>Get Rid of Events
for Users</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/recommended-user/"><span>Recommend
Users</span></a></li></ul></li><li class="level-2"><a class="expandible"
href="#"><span>Classification</span></a><ul><l
i class="level-3"><a class="final"
href="/templates/classification/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/classification/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/classification/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/classification/add-algorithm/"><span>Use Alternative
Algorithm</span></a></li><li class="level-3"><a class="final"
href="/templates/classification/reading-custom-properties/"><span>Read Custom
Properties</span></a></li></ul></li></ul></li><li class="level-1"><a
class="expandible" href="#"><span>Engine Template Gallery</span></a><ul><li
class="level-2"><a class="final"
href="/gallery/template-gallery/"><span>Browse</span></a></li><li
class="level-2"><a class="final"
href="/community/submit-template/"><span>Submit your Engine as a
Template</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><s
pan>Demo Tutorials</span></a><ul><li class="level-2"><a class="final"
href="/demo/tapster/"><span>Comics Recommendation Demo</span></a></li><li
class="level-2"><a class="final" href="/demo/community/"><span>Community
Contributed Demo</span></a></li><li class="level-2"><a class="final"
href="/demo/textclassification/"><span>Text Classification Engine
Tutorial</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="/community/"><span>Getting Involved</span></a><ul><li class="level-2"><a
class="final" href="/community/contribute-code/"><span>Contribute
Code</span></a></li><li class="level-2"><a class="final"
href="/community/contribute-documentation/"><span>Contribute
Documentation</span></a></li><li class="level-2"><a class="final"
href="/community/contribute-sdk/"><span>Contribute a SDK</span></a></li><li
class="level-2"><a class="final"
href="/community/contribute-webhook/"><span>Contribute a
Webhook</span></a></li><li class="level-2"><a class="final" href="/communi
ty/projects/"><span>Community Projects</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Getting
Help</span></a><ul><li class="level-2"><a class="final"
href="/resources/faq/"><span>FAQs</span></a></li><li class="level-2"><a
class="final" href="/support/"><span>Support</span></a></li></ul></li><li
class="level-1"><a class="expandible"
href="#"><span>Resources</span></a><ul><li class="level-2"><a class="final"
href="/cli/"><span>Command-line Interface</span></a></li><li class="level-2"><a
class="final" href="/resources/release/"><span>Release
Cadence</span></a></li><li class="level-2"><a class="final"
href="/resources/intellij/"><span>Developing Engines with IntelliJ
IDEA</span></a></li><li class="level-2"><a class="final"
href="/resources/upgrade/"><span>Upgrade Instructions</span></a></li><li
class="level-2"><a class="final"
href="/resources/glossary/"><span>Glossary</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Ap
ache Software Foundation</span></a><ul><li class="level-2"><a class="final"
href="https://www.apache.org/";><span>Apache Homepage</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/licenses/";><span>License</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/foundation/sponsorship.html";><span>Sponsorship</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/foundation/thanks.html";><span>Thanks</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/security/";><span>Security</span></a></li></ul></li></ul></nav></div><div
class="col-md-9 col-sm-12"><div class="content-header hidden-md
hidden-lg"><div id="breadcrumbs" class="hidden-sm hidden xs"><ul><li><a
href="#">ML Tuning and Evaluation</a><span
class="spacer">></span></li><li><span class="last">Hyperparameter
Tuning</span></li></ul></div><div id="page-title"><h1>Hyperparameter
Tuning</h1></div></div><div id="table-of-cont
ent-wrapper"><h5>On this page</h5><aside id="table-of-contents"><ul> <li> <a
href="#quick-start">Quick Start</a> </li> <li> <a
href="#detailed-explanation">Detailed Explanation</a> </li> <li> <a
href="#the-evaluation-design">The Evaluation Design</a> </li> <li> <a
href="#evaluation-data-generation">Evaluation Data Generation</a> </li> <li> <a
href="#evaluation-metrics">Evaluation Metrics</a> </li> <li> <a
href="#parameters-generation">Parameters Generation</a> </li> <li> <a
href="#running-the-evaluation">Running the Evaluation</a> </li> <li> <a
href="#notes">Notes</a> </li> </ul> </aside><hr/><a id="edit-page-link"
href="https://github.com/apache/incubator-predictionio/tree/livedoc/docs/manual/source/evaluation/paramtuning.html.md";><img
src="/images/icons/edit-pencil-d6c1bb3d.png"/>Edit this page</a></div><div
class="content-header hidden-sm hidden-xs"><div id="breadcrumbs"
class="hidden-sm hidden xs"><ul><li><a href="#">ML Tuning and
Evaluation</a><span class="spacer">></span></
li><li><span class="last">Hyperparameter Tuning</span></li></ul></div><div
id="page-title"><h1>Hyperparameter Tuning</h1></div></div><div class="content">
<p>A PredictionIO engine is instantiated by a set of parameters. These
parameters define which algorithm is to be used, as well supply the parameters
for the algorithm itself. This naturally raises the question of how to choose
the best set of parameters. The evaluation module streamlines the process of
<em>tuning</em> the engine to the best parameter set and deploys it.</p><h2
id='quick-start' class='header-anchors'>Quick Start</h2><p>We demonstrate the
evaluation with <a href="/templates/classification/quickstart/">the
classification template</a>. The classification template uses a naive bayesian
algorithm that has a smoothing parameter. We evaluate the prediction quality
against different parameter values to find the best parameter values, and then
deploy it.</p><h3 id='edit-the-appid' class='header-anchors'>Edit the AppId</h3>
<p>Edit
MyClassification/src/main/scala/<strong><em>Evaluation.scala</em></strong> to
specify the <em>appId</em> you used to import the data.</p><div
class="highlight scala"><table style="border-spacing: 0"><tbody><tr><td
class="gutter gl" style="text-align: right"><pre class="lineno">1
2
3
4
@@ -385,7 +385,7 @@ Metrics:
org.template.classification.Accuracy: 0.9281045751633987
The best variant params can be found <span class="k">in </span>best.json
<span class="o">[</span>INFO] <span class="o">[</span>CoreWorkflow<span
class="nv">$]</span> runEvaluation completed
-</pre></td></tr></tbody></table> </div> <h2 id='notes'
class='header-anchors'>Notes</h2> <ul> <li>We deliberately not metion
<strong><em>test set</em></strong> in this hyperparameter tuning guide. In
machine learning literature, the <strong><em>test set</em></strong> is a
separate piece of data which is used to evaluate the final engine params
outputted by the evaluation process. This guarantees that no information in the
training / validation set is <em>leaked</em> into the engine params and yields
a biased outcome. With PredictionIO, there are multiple ways of conducting
robust tuning, we will cover this topic in the coming sections.</li> </ul>
</div></div></div></div><footer><div class="container"><div
class="seperator"></div><div class="row"><div class="col-md-6
footer-link-column"><div
class="footer-link-column-row"><h4>Community</h4><ul><li><a
href="//predictionio.apache.org/install/"
target="blank">Download</a></li><li><a href="//predictionio.apache.org/"
target="blank">Docs<
/a></li><li><a href="//github.com/apache/incubator-predictionio"
target="blank">GitHub</a></li><li><a
href="mailto:[email protected]"; target="blank">Subscribe
to User Mailing List</a></li><li><a
href="//stackoverflow.com/questions/tagged/predictionio"
target="blank">Stackoverflow</a></li></ul></div></div><div class="col-md-6
footer-link-column"><div
class="footer-link-column-row"><h4>Contribute</h4><ul><li><a
href="//predictionio.apache.org/community/contribute-code/"
target="blank">Contribute</a></li><li><a
href="//github.com/apache/incubator-predictionio" target="blank">Source
Code</a></li><li><a href="//issues.apache.org/jira/browse/PIO"
target="blank">Bug Tracker</a></li><li><a
href="mailto:[email protected]"; target="blank">Subscribe to
Development Mailing List</a></li></ul></div></div></div><div class="row"><div
class="col-md-12 footer-link-column"><p>Apache PredictionIO, PredictionIO,
Apache, the Apache feather logo, and the Apache Pred
ictionIO project logo are either registered trademarks or trademarks of The
Apache Software Foundation in the United States and other countries.</p><p>All
other marks mentioned may be trademarks or registered trademarks of their
respective owners.</p></div></div></div><div id="footer-bottom"><div
class="container"><div class="row"><div class="col-md-12"><div
id="footer-logo-wrapper"><img alt="PredictionIO"
src="/images/logos/logo-white-d1e9c6e6.png"/><span>â¢</span></div><div
id="social-icons-wrapper"><a class="github-button"
href="https://github.com/apache/incubator-predictionio"; data-style="mega"
data-count-href="/apache/incubator-predictionio/stargazers"
data-count-api="/repos/apache/incubator-predictionio#stargazers_count"
data-count-aria-label="# stargazers on GitHub" aria-label="Star
apache/incubator-predictionio on GitHub">Star</a> <a class="github-button"
href="https://github.com/apache/incubator-predictionio/fork";
data-icon="octicon-git-branch" data-style="mega" data-count
-href="/apache/incubator-predictionio/network"
data-count-api="/repos/apache/incubator-predictionio#forks_count"
data-count-aria-label="# forks on GitHub" aria-label="Fork
apache/incubator-predictionio on GitHub">Fork</a> <script id="github-bjs"
async="" defer="" src="https://buttons.github.io/buttons.js";></script><a
href="https://twitter.com/predictionio"; target="blank"><img alt="PredictionIO
on Twitter" src="/images/icons/twitter-ea9dc152.png"/></a> <a
href="https://www.facebook.com/predictionio"; target="blank"><img
alt="PredictionIO on Facebook" src="/images/icons/facebook-5c57939c.png"/></a>
</div></div></div></div></div></footer></div><script>(function(w,d,t,u,n,s,e){w['SwiftypeObject']=n;w[n]=w[n]||function(){
+</pre></td></tr></tbody></table> </div> <h2 id='notes'
class='header-anchors'>Notes</h2> <ul> <li>We deliberately not metion
<strong><em>test set</em></strong> in this hyperparameter tuning guide. In
machine learning literature, the <strong><em>test set</em></strong> is a
separate piece of data which is used to evaluate the final engine params
outputted by the evaluation process. This guarantees that no information in the
training / validation set is <em>leaked</em> into the engine params and yields
a biased outcome. With PredictionIO, there are multiple ways of conducting
robust tuning, we will cover this topic in the coming sections.</li> </ul>
</div></div></div></div><footer><div class="container"><div
class="seperator"></div><div class="row"><div class="col-md-6
footer-link-column"><div
class="footer-link-column-row"><h4>Community</h4><ul><li><a
href="//predictionio.apache.org/install/"
target="blank">Download</a></li><li><a href="//predictionio.apache.org/"
target="blank">Docs<
/a></li><li><a href="//github.com/apache/incubator-predictionio"
target="blank">GitHub</a></li><li><a
href="mailto:[email protected]"; target="blank">Subscribe
to User Mailing List</a></li><li><a
href="//stackoverflow.com/questions/tagged/predictionio"
target="blank">Stackoverflow</a></li></ul></div></div><div class="col-md-6
footer-link-column"><div
class="footer-link-column-row"><h4>Contribute</h4><ul><li><a
href="//predictionio.apache.org/community/contribute-code/"
target="blank">Contribute</a></li><li><a
href="//github.com/apache/incubator-predictionio" target="blank">Source
Code</a></li><li><a href="//issues.apache.org/jira/browse/PIO"
target="blank">Bug Tracker</a></li><li><a
href="mailto:[email protected]"; target="blank">Subscribe to
Development Mailing List</a></li></ul></div></div></div><div class="row"><div
class="col-md-12 footer-link-column"><p>Apache PredictionIO, PredictionIO,
Apache, the Apache feather logo, and the Apache Pred
ictionIO project logo are either registered trademarks or trademarks of The
Apache Software Foundation in the United States and other countries.</p><p>All
other marks mentioned may be trademarks or registered trademarks of their
respective owners.</p></div></div></div><div id="footer-bottom"><div
class="container"><div class="row"><div class="col-md-12"><div
id="footer-logo-wrapper"><img alt="PredictionIO"
src="/images/logos/logo-white-d1e9c6e6.png"/><span>®</span></div><div
id="social-icons-wrapper"><a class="github-button"
href="https://github.com/apache/incubator-predictionio"; data-style="mega"
data-count-href="/apache/incubator-predictionio/stargazers"
data-count-api="/repos/apache/incubator-predictionio#stargazers_count"
data-count-aria-label="# stargazers on GitHub" aria-label="Star
apache/incubator-predictionio on GitHub">Star</a> <a class="github-button"
href="https://github.com/apache/incubator-predictionio/fork";
data-icon="octicon-git-branch" data-style="mega" data-count-
href="/apache/incubator-predictionio/network"
data-count-api="/repos/apache/incubator-predictionio#forks_count"
data-count-aria-label="# forks on GitHub" aria-label="Fork
apache/incubator-predictionio on GitHub">Fork</a> <script id="github-bjs"
async="" defer="" src="https://buttons.github.io/buttons.js";></script><a
href="https://twitter.com/predictionio"; target="blank"><img alt="PredictionIO
on Twitter" src="/images/icons/twitter-ea9dc152.png"/></a> <a
href="https://www.facebook.com/predictionio"; target="blank"><img
alt="PredictionIO on Facebook" src="/images/icons/facebook-5c57939c.png"/></a>
</div></div></div></div></div></footer></div><script>(function(w,d,t,u,n,s,e){w['SwiftypeObject']=n;w[n]=w[n]||function(){
(w[n].q=w[n].q||[]).push(arguments);};s=d.createElement(t);
e=d.getElementsByTagName(t)[0];s.async=1;s.src=u;e.parentNode.insertBefore(s,e);
})(window,document,'script','//s.swiftypecdn.com/install/v1/st.js','_st');
