http://git-wip-us.apache.org/repos/asf/predictionio-site/blob/6e02047e/system/index.html
----------------------------------------------------------------------
diff --git a/system/index.html b/system/index.html
index 22524a3..5c2c5e3 100644
--- a/system/index.html
+++ b/system/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html><head><title>System Architecture and
Dependencies</title><meta charset="utf-8"/><meta content="IE=edge,chrome=1"
http-equiv="X-UA-Compatible"/><meta name="viewport"
content="width=device-width, initial-scale=1.0"/><meta class="swiftype"
name="title" data-type="string" content="System Architecture and
Dependencies"/><link rel="canonical"
href="https://predictionio.apache.org/system/"/><link
href="/images/favicon/normal-b330020a.png" rel="shortcut icon"/><link
href="/images/favicon/apple-c0febcf2.png" rel="apple-touch-icon"/><link
href="//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800"
rel="stylesheet"/><link
href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css"
rel="stylesheet"/><link href="/stylesheets/application-eccfc6cb.css"
rel="stylesheet" type="text/css"/><script
src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.2/html5shiv.min.js"></script><script
src="//cdn.mathjax.
org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script><script
src="//use.typekit.net/pqo0itb.js"></script><script>try{Typekit.load({ async:
true });}catch(e){}</script></head><body><div id="global"><header><div
class="container" id="header-wrapper"><div class="row"><div
class="col-sm-12"><div id="logo-wrapper"><span id="drawer-toggle"></span><a
href="#"></a><a href="http://predictionio.apache.org/";><img alt="Apache
PredictionIO" id="logo"
src="/images/logos/logo-ee2b9bb3.png"/></a><span>®</span></div><div
id="menu-wrapper"><div id="pill-wrapper"><a class="pill left"
href="/gallery/template-gallery">TEMPLATES</a> <a class="pill right"
href="//github.com/apache/predictionio/">OPEN SOURCE</a></div></div><img
class="mobile-search-bar-toggler hidden-md hidden-lg"
src="/images/icons/search-glass-704bd4ff.png"/></div></div></div></header><div
id="search-bar-row-wrapper"><div class="container-fluid"
id="search-bar-row"><div class="row"><div class="col-md-9 col-sm-11 col-xs-1
1"><div class="hidden-md hidden-lg"
id="mobile-page-heading-wrapper"><p>PredictionIO Docs</p><h4>Architecture
Overview</h4></div><h4 class="hidden-sm hidden-xs">PredictionIO
Docs</h4></div><div class="col-md-3 col-sm-1 col-xs-1 hidden-md hidden-lg"><img
id="left-menu-indicator"
src="/images/icons/down-arrow-dfe9f7fe.png"/></div><div class="col-md-3
col-sm-12 col-xs-12 swiftype-wrapper"><div class="swiftype"><form
class="search-form"><img class="search-box-toggler hidden-xs hidden-sm"
src="/images/icons/search-glass-704bd4ff.png"/><div class="search-box"><img
src="/images/icons/search-glass-704bd4ff.png"/><input type="text"
id="st-search-input" class="st-search-input" placeholder="Search
Doc..."/></div><img class="swiftype-row-hider hidden-md hidden-lg"
src="/images/icons/drawer-toggle-active-fcbef12a.png"/></form></div></div><div
class="mobile-left-menu-toggler hidden-md
hidden-lg"></div></div></div></div><div id="page" class="container-fluid"><div
class="row"><div id="left-menu-wra
pper" class="col-md-3"><nav id="nav-main"><ul><li class="level-1"><a
class="expandible" href="/"><span>Apache PredictionIO®
Documentation</span></a><ul><li class="level-2"><a class="final"
href="/"><span>Welcome to Apache PredictionIO®</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Getting
Started</span></a><ul><li class="level-2"><a class="final"
href="/start/"><span>A Quick Intro</span></a></li><li class="level-2"><a
class="final" href="/install/"><span>Installing Apache
PredictionIO</span></a></li><li class="level-2"><a class="final"
href="/start/download/"><span>Downloading an Engine Template</span></a></li><li
class="level-2"><a class="final" href="/start/deploy/"><span>Deploying Your
First Engine</span></a></li><li class="level-2"><a class="final"
href="/start/customize/"><span>Customizing the
Engine</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Integrating with Your App</span></a><ul><li class="level-2">
<a class="final" href="/appintegration/"><span>App Integration
Overview</span></a></li><li class="level-2"><a class="expandible"
href="/sdk/"><span>List of SDKs</span></a><ul><li class="level-3"><a
class="final" href="/sdk/java/"><span>Java & Android SDK</span></a></li><li
class="level-3"><a class="final" href="/sdk/php/"><span>PHP
SDK</span></a></li><li class="level-3"><a class="final"
href="/sdk/python/"><span>Python SDK</span></a></li><li class="level-3"><a
class="final" href="/sdk/ruby/"><span>Ruby SDK</span></a></li><li
class="level-3"><a class="final" href="/sdk/community/"><span>Community Powered
SDKs</span></a></li></ul></li></ul></li><li class="level-1"><a
class="expandible" href="#"><span>Deploying an Engine</span></a><ul><li
class="level-2"><a class="final" href="/deploy/"><span>Deploying as a Web
Service</span></a></li><li class="level-2"><a class="final"
href="/batchpredict/"><span>Batch Predictions</span></a></li><li
class="level-2"><a class="final" href="/deploy/monit
oring/"><span>Monitoring Engine</span></a></li><li class="level-2"><a
class="final" href="/deploy/engineparams/"><span>Setting Engine
Parameters</span></a></li><li class="level-2"><a class="final"
href="/deploy/enginevariants/"><span>Deploying Multiple Engine
Variants</span></a></li><li class="level-2"><a class="final"
href="/deploy/plugin/"><span>Engine Server Plugin</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Customizing an
Engine</span></a><ul><li class="level-2"><a class="final"
href="/customize/"><span>Learning DASE</span></a></li><li class="level-2"><a
class="final" href="/customize/dase/"><span>Implement DASE</span></a></li><li
class="level-2"><a class="final"
href="/customize/troubleshooting/"><span>Troubleshooting Engine
Development</span></a></li><li class="level-2"><a class="final"
href="/api/current/#package"><span>Engine Scala
APIs</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Collecting and Analy
zing Data</span></a><ul><li class="level-2"><a class="final"
href="/datacollection/"><span>Event Server Overview</span></a></li><li
class="level-2"><a class="final"
href="/datacollection/eventapi/"><span>Collecting Data with
REST/SDKs</span></a></li><li class="level-2"><a class="final"
href="/datacollection/eventmodel/"><span>Events Modeling</span></a></li><li
class="level-2"><a class="final"
href="/datacollection/webhooks/"><span>Unifying Multichannel Data with
Webhooks</span></a></li><li class="level-2"><a class="final"
href="/datacollection/channel/"><span>Channel</span></a></li><li
class="level-2"><a class="final"
href="/datacollection/batchimport/"><span>Importing Data in
Batch</span></a></li><li class="level-2"><a class="final"
href="/datacollection/analytics/"><span>Using Analytics
Tools</span></a></li><li class="level-2"><a class="final"
href="/datacollection/plugin/"><span>Event Server
Plugin</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>
Choosing an Algorithm</span></a><ul><li class="level-2"><a class="final"
href="/algorithm/"><span>Built-in Algorithm Libraries</span></a></li><li
class="level-2"><a class="final" href="/algorithm/switch/"><span>Switching to
Another Algorithm</span></a></li><li class="level-2"><a class="final"
href="/algorithm/multiple/"><span>Combining Multiple
Algorithms</span></a></li><li class="level-2"><a class="final"
href="/algorithm/custom/"><span>Adding Your Own
Algorithms</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Tuning and Evaluation</span></a><ul><li class="level-2"><a
class="final" href="/evaluation/"><span>Overview</span></a></li><li
class="level-2"><a class="final"
href="/evaluation/paramtuning/"><span>Hyperparameter Tuning</span></a></li><li
class="level-2"><a class="final"
href="/evaluation/evaluationdashboard/"><span>Evaluation
Dashboard</span></a></li><li class="level-2"><a class="final"
href="/evaluation/metricchoose/"><span>Choosing Evaluat
ion Metrics</span></a></li><li class="level-2"><a class="final"
href="/evaluation/metricbuild/"><span>Building Evaluation
Metrics</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>System Architecture</span></a><ul><li class="level-2"><a
class="final active" href="/system/"><span>Architecture
Overview</span></a></li><li class="level-2"><a class="final"
href="/system/anotherdatastore/"><span>Using Another Data
Store</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>PredictionIO® Official Templates</span></a><ul><li
class="level-2"><a class="final"
href="/templates/"><span>Intro</span></a></li><li class="level-2"><a
class="expandible" href="#"><span>Recommendation</span></a><ul><li
class="level-3"><a class="final"
href="/templates/recommendation/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="f
inal" href="/templates/recommendation/evaluation/"><span>Evaluation
Explained</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/recommendation/reading-custom-events/"><span>Read Custom
Events</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/customize-data-prep/"><span>Customize Data
Preparator</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/customize-serving/"><span>Customize
Serving</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/training-with-implicit-preference/"><span>Train
with Implicit Preference</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/blacklist-items/"><span>Filter Recommended
Items by Blacklist in Query</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/batch-evaluator/">
<span>Batch Persistable Evaluator</span></a></li></ul></li><li
class="level-2"><a class="expandible" href="#"><span>E-Commerce
Recommendation</span></a><ul><li class="level-3"><a class="final"
href="/templates/ecommercerecommendation/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/ecommercerecommendation/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/ecommercerecommendation/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/ecommercerecommendation/train-with-rate-event/"><span>Train
with Rate Event</span></a></li><li class="level-3"><a class="final"
href="/templates/ecommercerecommendation/adjust-score/"><span>Adjust
Score</span></a></li></ul></li><li class="level-2"><a class="expandible"
href="#"><span>Similar Product</span></a><ul><li class="level-3"><a
class="final" href="/templates/similarproduct/quickstart/"><span>Quick
Start</span></a></li><li cla
ss="level-3"><a class="final"
href="/templates/similarproduct/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/similarproduct/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/similarproduct/multi-events-multi-algos/"><span>Multiple
Events and Multiple Algorithms</span></a></li><li class="level-3"><a
class="final"
href="/templates/similarproduct/return-item-properties/"><span>Returns Item
Properties</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/train-with-rate-event/"><span>Train with Rate
Event</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/rid-user-set-event/"><span>Get Rid of Events
for Users</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/recommended-user/"><span>Recommend
Users</span></a></li></ul></li><li class="level-2"><a class="expandible"
href="#"><span>Classification</span></a><ul><li
class="level-3"><a class="final"
href="/templates/classification/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/classification/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/classification/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/classification/add-algorithm/"><span>Use Alternative
Algorithm</span></a></li><li class="level-3"><a class="final"
href="/templates/classification/reading-custom-properties/"><span>Read Custom
Properties</span></a></li></ul></li></ul></li><li class="level-1"><a
class="expandible" href="#"><span>Engine Template Gallery</span></a><ul><li
class="level-2"><a class="final"
href="/gallery/template-gallery/"><span>Browse</span></a></li><li
class="level-2"><a class="final"
href="/community/submit-template/"><span>Submit your Engine as a
Template</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><spa
n>Demo Tutorials</span></a><ul><li class="level-2"><a class="final"
href="/demo/tapster/"><span>Comics Recommendation Demo</span></a></li><li
class="level-2"><a class="final" href="/demo/community/"><span>Community
Contributed Demo</span></a></li><li class="level-2"><a class="final"
href="/demo/textclassification/"><span>Text Classification Engine
Tutorial</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="/community/"><span>Getting Involved</span></a><ul><li class="level-2"><a
class="final" href="/community/contribute-code/"><span>Contribute
Code</span></a></li><li class="level-2"><a class="final"
href="/community/contribute-documentation/"><span>Contribute
Documentation</span></a></li><li class="level-2"><a class="final"
href="/community/contribute-sdk/"><span>Contribute a SDK</span></a></li><li
class="level-2"><a class="final"
href="/community/contribute-webhook/"><span>Contribute a
Webhook</span></a></li><li class="level-2"><a class="final" href="/community
/projects/"><span>Community Projects</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Getting
Help</span></a><ul><li class="level-2"><a class="final"
href="/resources/faq/"><span>FAQs</span></a></li><li class="level-2"><a
class="final" href="/support/"><span>Support</span></a></li></ul></li><li
class="level-1"><a class="expandible"
href="#"><span>Resources</span></a><ul><li class="level-2"><a class="final"
href="/cli/"><span>Command-line Interface</span></a></li><li class="level-2"><a
class="final" href="/resources/release/"><span>Release
Cadence</span></a></li><li class="level-2"><a class="final"
href="/resources/intellij/"><span>Developing Engines with IntelliJ
IDEA</span></a></li><li class="level-2"><a class="final"
href="/resources/upgrade/"><span>Upgrade Instructions</span></a></li><li
class="level-2"><a class="final"
href="/resources/glossary/"><span>Glossary</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Apac
he Software Foundation</span></a><ul><li class="level-2"><a class="final"
href="https://www.apache.org/";><span>Apache Homepage</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/licenses/";><span>License</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/foundation/sponsorship.html";><span>Sponsorship</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/foundation/thanks.html";><span>Thanks</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/security/";><span>Security</span></a></li></ul></li></ul></nav></div><div
class="col-md-9 col-sm-12"><div class="content-header hidden-md
hidden-lg"><div id="breadcrumbs" class="hidden-sm hidden xs"><ul><li><a
href="#">System Architecture</a><span class="spacer">></span></li><li><span
class="last">Architecture Overview</span></li></ul></div><div
id="page-title"><h1>System Architecture and Dependencies</h1></div></div><div
id="table
-of-content-wrapper"><a id="edit-page-link"
href="https://github.com/apache/predictionio/tree/livedoc/docs/manual/source/system/index.html.md";><img
src="/images/icons/edit-pencil-d6c1bb3d.png"/>Edit this page</a></div><div
class="content-header hidden-sm hidden-xs"><div id="breadcrumbs"
class="hidden-sm hidden xs"><ul><li><a href="#">System Architecture</a><span
class="spacer">></span></li><li><span class="last">Architecture
Overview</span></li></ul></div><div id="page-title"><h1>System Architecture and
Dependencies</h1></div></div><div class="content"> <p>During the <a
href="/install">installation</a>, you have installed the latest stable versions
of the following software:</p> <ul> <li>Apache Hadoop up to 2.7.2 (required
only if YARN and HDFS are needed)</li> <li>Apache HBase up to 1.2.4</li>
<li>Apache Spark up to 1.6.3, or up to 2.3.1</li> <li>Elasticsearch up to
1.7.5, or up to 5.6.11</li> </ul> <p>This section explains general
rules-of-thumb for how they are used in Predict
ionIO. The actual implementation of the Template will define how much of this
applies. PredictionIO is flexible about much of this configuration but its
Templates generally fit the Lambda model for integrating real-time serving with
background periodic model updates.</p><p><img alt="PredictionIO Systems"
src="/images/pio-architecture-a1e2a38c.svg"/></p><p><strong>HBase</strong>:
Event Server uses Apache HBase (or JDBC DB for small data) as the data store.
It stores imported events. If you are not using the PredictionIO Event Server,
you do not need to install HBase.</p><p><strong>Apache Spark</strong>: Spark is
a large-scale data processing engine that powers the data preparation and input
to the algorithm, training, and sometimes the serving processing. PredictionIO
allows for different engines to be used in training but many algorithms come
from Spark's MLlib.</p><p><strong>HDFS</strong>: is a distributed
filesystem from Hadoop. It allows storage to be shared among clustered m
achines. It is used to stage data for batch import into PredictionIO, for
export of Event Server datasets, and for storage of some models (see your
template for details).</p><p>The output of training has two parts: a model and
its meta-data. The model is then stored in HDFS, a local file system, or
Elasticsearch. See the details of your
algorithm.</p><p><strong>Elasticsearch</strong>: stores metadata such as model
versions, engine versions, access key and app ID mappings, evaluation results,
etc. For some templates it may store the
model.</p></div></div></div></div><footer><div class="container"><div
class="seperator"></div><div class="row"><div class="col-md-6
footer-link-column"><div
class="footer-link-column-row"><h4>Community</h4><ul><li><a
href="//predictionio.apache.org/install/"
target="blank">Download</a></li><li><a href="//predictionio.apache.org/"
target="blank">Docs</a></li><li><a href="//github.com/apache/predictionio"
target="blank">GitHub</a></li><li><a href="mailto:us
[email protected]" target="blank">Subscribe to User Mailing
List</a></li><li><a href="//stackoverflow.com/questions/tagged/predictionio"
target="blank">Stackoverflow</a></li></ul></div></div><div class="col-md-6
footer-link-column"><div
class="footer-link-column-row"><h4>Contribute</h4><ul><li><a
href="//predictionio.apache.org/community/contribute-code/"
target="blank">Contribute</a></li><li><a
href="//github.com/apache/predictionio" target="blank">Source
Code</a></li><li><a href="//issues.apache.org/jira/browse/PIO"
target="blank">Bug Tracker</a></li><li><a
href="mailto:[email protected]"; target="blank">Subscribe to
Development Mailing List</a></li></ul></div></div></div><div class="row"><div
class="col-md-12 footer-link-column"><p>Apache PredictionIO, PredictionIO,
Apache, the Apache feather logo, and the Apache PredictionIO project logo are
either registered trademarks or trademarks of The Apache Software Foundation in
the United States and
other countries.</p><p>All other marks mentioned may be trademarks or
registered trademarks of their respective owners.</p></div></div></div><div
id="footer-bottom"><div class="container"><div class="row"><div
class="col-md-12"><div id="footer-logo-wrapper"><img alt="PredictionIO"
src="/images/logos/logo-white-d1e9c6e6.png"/><span>®</span></div><div
id="social-icons-wrapper"><a class="github-button"
href="https://github.com/apache/predictionio"; data-icon="octicon-star"
data-show-count="true" aria-label="Star apache/predictionio on GitHub">Star</a>
<a class="github-button" href="https://github.com/apache/predictionio/fork";
data-icon="octicon-repo-forked" data-show-count="true" aria-label="Fork
apache/predictionio on GitHub">Fork</a> <script id="github-bjs" async=""
defer="" src="https://buttons.github.io/buttons.js";></script><a
href="https://twitter.com/predictionio"; target="blank"><img alt="PredictionIO
on Twitter" src="/images/icons/twitter-ea9dc152.png"/></a> <a href="https://ww
w.facebook.com/predictionio" target="blank"><img alt="PredictionIO on
Facebook" src="/images/icons/facebook-5c57939c.png"/></a>
</div></div></div></div></div></footer></div><script>(function(w,d,t,u,n,s,e){w['SwiftypeObject']=n;w[n]=w[n]||function(){
+<!DOCTYPE html><html><head><title>System Architecture and
Dependencies</title><meta charset="utf-8"/><meta content="IE=edge,chrome=1"
http-equiv="X-UA-Compatible"/><meta name="viewport"
content="width=device-width, initial-scale=1.0"/><meta class="swiftype"
name="title" data-type="string" content="System Architecture and
Dependencies"/><link rel="canonical"
href="https://predictionio.apache.org/system/"/><link
href="/images/favicon/normal-b330020a.png" rel="shortcut icon"/><link
href="/images/favicon/apple-c0febcf2.png" rel="apple-touch-icon"/><link
href="//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800"
rel="stylesheet"/><link
href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css"
rel="stylesheet"/><link href="/stylesheets/application-eccfc6cb.css"
rel="stylesheet" type="text/css"/><script
src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.2/html5shiv.min.js"></script><script
src="//cdn.mathjax.
org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script><script
src="//use.typekit.net/pqo0itb.js"></script><script>try{Typekit.load({ async:
true });}catch(e){}</script></head><body><div id="global"><header><div
class="container" id="header-wrapper"><div class="row"><div
class="col-sm-12"><div id="logo-wrapper"><span id="drawer-toggle"></span><a
href="#"></a><a href="http://predictionio.apache.org/";><img alt="Apache
PredictionIO" id="logo"
src="/images/logos/logo-ee2b9bb3.png"/></a><span>®</span></div><div
id="menu-wrapper"><div id="pill-wrapper"><a class="pill left"
href="/gallery/template-gallery">TEMPLATES</a> <a class="pill right"
href="//github.com/apache/predictionio/">OPEN SOURCE</a></div></div><img
class="mobile-search-bar-toggler hidden-md hidden-lg"
src="/images/icons/search-glass-704bd4ff.png"/></div></div></div></header><div
id="search-bar-row-wrapper"><div class="container-fluid"
id="search-bar-row"><div class="row"><div class="col-md-9 col-sm-11 col-xs-1
1"><div class="hidden-md hidden-lg"
id="mobile-page-heading-wrapper"><p>PredictionIO Docs</p><h4>Architecture
Overview</h4></div><h4 class="hidden-sm hidden-xs">PredictionIO
Docs</h4></div><div class="col-md-3 col-sm-1 col-xs-1 hidden-md hidden-lg"><img
id="left-menu-indicator"
src="/images/icons/down-arrow-dfe9f7fe.png"/></div><div class="col-md-3
col-sm-12 col-xs-12 swiftype-wrapper"><div class="swiftype"><form
class="search-form"><img class="search-box-toggler hidden-xs hidden-sm"
src="/images/icons/search-glass-704bd4ff.png"/><div class="search-box"><img
src="/images/icons/search-glass-704bd4ff.png"/><input type="text"
id="st-search-input" class="st-search-input" placeholder="Search
Doc..."/></div><img class="swiftype-row-hider hidden-md hidden-lg"
src="/images/icons/drawer-toggle-active-fcbef12a.png"/></form></div></div><div
class="mobile-left-menu-toggler hidden-md
hidden-lg"></div></div></div></div><div id="page" class="container-fluid"><div
class="row"><div id="left-menu-wra
pper" class="col-md-3"><nav id="nav-main"><ul><li class="level-1"><a
class="expandible" href="/"><span>Apache PredictionIO®
Documentation</span></a><ul><li class="level-2"><a class="final"
href="/"><span>Welcome to Apache PredictionIO®</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Getting
Started</span></a><ul><li class="level-2"><a class="final"
href="/start/"><span>A Quick Intro</span></a></li><li class="level-2"><a
class="final" href="/install/"><span>Installing Apache
PredictionIO</span></a></li><li class="level-2"><a class="final"
href="/start/download/"><span>Downloading an Engine Template</span></a></li><li
class="level-2"><a class="final" href="/start/deploy/"><span>Deploying Your
First Engine</span></a></li><li class="level-2"><a class="final"
href="/start/customize/"><span>Customizing the
Engine</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Integrating with Your App</span></a><ul><li class="level-2">
<a class="final" href="/appintegration/"><span>App Integration
Overview</span></a></li><li class="level-2"><a class="expandible"
href="/sdk/"><span>List of SDKs</span></a><ul><li class="level-3"><a
class="final" href="/sdk/java/"><span>Java & Android SDK</span></a></li><li
class="level-3"><a class="final" href="/sdk/php/"><span>PHP
SDK</span></a></li><li class="level-3"><a class="final"
href="/sdk/python/"><span>Python SDK</span></a></li><li class="level-3"><a
class="final" href="/sdk/ruby/"><span>Ruby SDK</span></a></li><li
class="level-3"><a class="final" href="/sdk/community/"><span>Community Powered
SDKs</span></a></li></ul></li></ul></li><li class="level-1"><a
class="expandible" href="#"><span>Deploying an Engine</span></a><ul><li
class="level-2"><a class="final" href="/deploy/"><span>Deploying as a Web
Service</span></a></li><li class="level-2"><a class="final"
href="/batchpredict/"><span>Batch Predictions</span></a></li><li
class="level-2"><a class="final" href="/deploy/monit
oring/"><span>Monitoring Engine</span></a></li><li class="level-2"><a
class="final" href="/deploy/engineparams/"><span>Setting Engine
Parameters</span></a></li><li class="level-2"><a class="final"
href="/deploy/enginevariants/"><span>Deploying Multiple Engine
Variants</span></a></li><li class="level-2"><a class="final"
href="/deploy/plugin/"><span>Engine Server Plugin</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Customizing an
Engine</span></a><ul><li class="level-2"><a class="final"
href="/customize/"><span>Learning DASE</span></a></li><li class="level-2"><a
class="final" href="/customize/dase/"><span>Implement DASE</span></a></li><li
class="level-2"><a class="final"
href="/customize/troubleshooting/"><span>Troubleshooting Engine
Development</span></a></li><li class="level-2"><a class="final"
href="/api/current/#package"><span>Engine Scala
APIs</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Collecting and Analy
zing Data</span></a><ul><li class="level-2"><a class="final"
href="/datacollection/"><span>Event Server Overview</span></a></li><li
class="level-2"><a class="final"
href="/datacollection/eventapi/"><span>Collecting Data with
REST/SDKs</span></a></li><li class="level-2"><a class="final"
href="/datacollection/eventmodel/"><span>Events Modeling</span></a></li><li
class="level-2"><a class="final"
href="/datacollection/webhooks/"><span>Unifying Multichannel Data with
Webhooks</span></a></li><li class="level-2"><a class="final"
href="/datacollection/channel/"><span>Channel</span></a></li><li
class="level-2"><a class="final"
href="/datacollection/batchimport/"><span>Importing Data in
Batch</span></a></li><li class="level-2"><a class="final"
href="/datacollection/analytics/"><span>Using Analytics
Tools</span></a></li><li class="level-2"><a class="final"
href="/datacollection/plugin/"><span>Event Server
Plugin</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>
Choosing an Algorithm</span></a><ul><li class="level-2"><a class="final"
href="/algorithm/"><span>Built-in Algorithm Libraries</span></a></li><li
class="level-2"><a class="final" href="/algorithm/switch/"><span>Switching to
Another Algorithm</span></a></li><li class="level-2"><a class="final"
href="/algorithm/multiple/"><span>Combining Multiple
Algorithms</span></a></li><li class="level-2"><a class="final"
href="/algorithm/custom/"><span>Adding Your Own
Algorithms</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Tuning and Evaluation</span></a><ul><li class="level-2"><a
class="final" href="/evaluation/"><span>Overview</span></a></li><li
class="level-2"><a class="final"
href="/evaluation/paramtuning/"><span>Hyperparameter Tuning</span></a></li><li
class="level-2"><a class="final"
href="/evaluation/evaluationdashboard/"><span>Evaluation
Dashboard</span></a></li><li class="level-2"><a class="final"
href="/evaluation/metricchoose/"><span>Choosing Evaluat
ion Metrics</span></a></li><li class="level-2"><a class="final"
href="/evaluation/metricbuild/"><span>Building Evaluation
Metrics</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>System Architecture</span></a><ul><li class="level-2"><a
class="final active" href="/system/"><span>Architecture
Overview</span></a></li><li class="level-2"><a class="final"
href="/system/anotherdatastore/"><span>Using Another Data
Store</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>PredictionIO® Official Templates</span></a><ul><li
class="level-2"><a class="final"
href="/templates/"><span>Intro</span></a></li><li class="level-2"><a
class="expandible" href="#"><span>Recommendation</span></a><ul><li
class="level-3"><a class="final"
href="/templates/recommendation/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="f
inal" href="/templates/recommendation/evaluation/"><span>Evaluation
Explained</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/recommendation/reading-custom-events/"><span>Read Custom
Events</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/customize-data-prep/"><span>Customize Data
Preparator</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/customize-serving/"><span>Customize
Serving</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/training-with-implicit-preference/"><span>Train
with Implicit Preference</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/blacklist-items/"><span>Filter Recommended
Items by Blacklist in Query</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/batch-evaluator/">
<span>Batch Persistable Evaluator</span></a></li></ul></li><li
class="level-2"><a class="expandible" href="#"><span>E-Commerce
Recommendation</span></a><ul><li class="level-3"><a class="final"
href="/templates/ecommercerecommendation/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/ecommercerecommendation/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/ecommercerecommendation/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/ecommercerecommendation/train-with-rate-event/"><span>Train
with Rate Event</span></a></li><li class="level-3"><a class="final"
href="/templates/ecommercerecommendation/adjust-score/"><span>Adjust
Score</span></a></li></ul></li><li class="level-2"><a class="expandible"
href="#"><span>Similar Product</span></a><ul><li class="level-3"><a
class="final" href="/templates/similarproduct/quickstart/"><span>Quick
Start</span></a></li><li cla
ss="level-3"><a class="final"
href="/templates/similarproduct/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/similarproduct/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/similarproduct/multi-events-multi-algos/"><span>Multiple
Events and Multiple Algorithms</span></a></li><li class="level-3"><a
class="final"
href="/templates/similarproduct/return-item-properties/"><span>Returns Item
Properties</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/train-with-rate-event/"><span>Train with Rate
Event</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/rid-user-set-event/"><span>Get Rid of Events
for Users</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/recommended-user/"><span>Recommend
Users</span></a></li></ul></li><li class="level-2"><a class="expandible"
href="#"><span>Classification</span></a><ul><li
class="level-3"><a class="final"
href="/templates/classification/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/classification/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/classification/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/classification/add-algorithm/"><span>Use Alternative
Algorithm</span></a></li><li class="level-3"><a class="final"
href="/templates/classification/reading-custom-properties/"><span>Read Custom
Properties</span></a></li></ul></li></ul></li><li class="level-1"><a
class="expandible" href="#"><span>Engine Template Gallery</span></a><ul><li
class="level-2"><a class="final"
href="/gallery/template-gallery/"><span>Browse</span></a></li><li
class="level-2"><a class="final"
href="/community/submit-template/"><span>Submit your Engine as a
Template</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><spa
n>Demo Tutorials</span></a><ul><li class="level-2"><a class="final"
href="/demo/tapster/"><span>Comics Recommendation Demo</span></a></li><li
class="level-2"><a class="final" href="/demo/community/"><span>Community
Contributed Demo</span></a></li><li class="level-2"><a class="final"
href="/demo/textclassification/"><span>Text Classification Engine
Tutorial</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="/community/"><span>Getting Involved</span></a><ul><li class="level-2"><a
class="final" href="/community/contribute-code/"><span>Contribute
Code</span></a></li><li class="level-2"><a class="final"
href="/community/contribute-documentation/"><span>Contribute
Documentation</span></a></li><li class="level-2"><a class="final"
href="/community/contribute-sdk/"><span>Contribute a SDK</span></a></li><li
class="level-2"><a class="final"
href="/community/contribute-webhook/"><span>Contribute a
Webhook</span></a></li><li class="level-2"><a class="final" href="/community
/projects/"><span>Community Projects</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Getting
Help</span></a><ul><li class="level-2"><a class="final"
href="/resources/faq/"><span>FAQs</span></a></li><li class="level-2"><a
class="final" href="/support/"><span>Support</span></a></li></ul></li><li
class="level-1"><a class="expandible"
href="#"><span>Resources</span></a><ul><li class="level-2"><a class="final"
href="/cli/"><span>Command-line Interface</span></a></li><li class="level-2"><a
class="final" href="/resources/release/"><span>Release
Cadence</span></a></li><li class="level-2"><a class="final"
href="/resources/intellij/"><span>Developing Engines with IntelliJ
IDEA</span></a></li><li class="level-2"><a class="final"
href="/resources/upgrade/"><span>Upgrade Instructions</span></a></li><li
class="level-2"><a class="final"
href="/resources/glossary/"><span>Glossary</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Apac
he Software Foundation</span></a><ul><li class="level-2"><a class="final"
href="https://www.apache.org/";><span>Apache Homepage</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/licenses/";><span>License</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/foundation/sponsorship.html";><span>Sponsorship</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/foundation/thanks.html";><span>Thanks</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/security/";><span>Security</span></a></li></ul></li></ul></nav></div><div
class="col-md-9 col-sm-12"><div class="content-header hidden-md
hidden-lg"><div id="breadcrumbs" class="hidden-sm hidden xs"><ul><li><a
href="#">System Architecture</a><span class="spacer">></span></li><li><span
class="last">Architecture Overview</span></li></ul></div><div
id="page-title"><h1>System Architecture and Dependencies</h1></div></div><div
id="table
-of-content-wrapper"><a id="edit-page-link"
href="https://github.com/apache/predictionio/tree/livedoc/docs/manual/source/system/index.html.md";><img
src="/images/icons/edit-pencil-d6c1bb3d.png"/>Edit this page</a></div><div
class="content-header hidden-sm hidden-xs"><div id="breadcrumbs"
class="hidden-sm hidden xs"><ul><li><a href="#">System Architecture</a><span
class="spacer">></span></li><li><span class="last">Architecture
Overview</span></li></ul></div><div id="page-title"><h1>System Architecture and
Dependencies</h1></div></div><div class="content"> <p>This section explains
general rules-of-thumb for how they are used in PredictionIO. The actual
implementation of the Template will define how much of this applies.
PredictionIO is flexible about much of this configuration but its Templates
generally fit the Lambda model for integrating real-time serving with
background periodic model updates.</p><p><img alt="PredictionIO Systems"
src="/images/pio-architecture-a1e2a38c.svg"/></p
><p><strong>HBase</strong>: Event Server uses Apache HBase (or JDBC DB for
>small data) as the data store. It stores imported events. If you are not
>using the PredictionIO Event Server, you do not need to install
>HBase.</p><p><strong>Apache Spark</strong>: Spark is a large-scale data
>processing engine that powers the data preparation and input to the
>algorithm, training, and sometimes the serving processing. PredictionIO
>allows for different engines to be used in training but many algorithms come
>from Spark's MLlib.</p><p><strong>HDFS</strong>: is a distributed
>filesystem from Hadoop. It allows storage to be shared among clustered
>machines. It is used to stage data for batch import into PredictionIO, for
>export of Event Server datasets, and for storage of some models (see your
>template for details).</p><p>The output of training has two parts: a model
>and its meta-data. The model is then stored in HDFS, a local file system, or
>Elasticsearch. See the details of your algorithm.</p><
p><strong>Elasticsearch</strong>: stores metadata such as model versions,
engine versions, access key and app ID mappings, evaluation results, etc. For
some templates it may store the model.</p></div></div></div></div><footer><div
class="container"><div class="seperator"></div><div class="row"><div
class="col-md-6 footer-link-column"><div
class="footer-link-column-row"><h4>Community</h4><ul><li><a
href="//predictionio.apache.org/install/"
target="blank">Download</a></li><li><a href="//predictionio.apache.org/"
target="blank">Docs</a></li><li><a href="//github.com/apache/predictionio"
target="blank">GitHub</a></li><li><a
href="mailto:[email protected]"; target="blank">Subscribe
to User Mailing List</a></li><li><a
href="//stackoverflow.com/questions/tagged/predictionio"
target="blank">Stackoverflow</a></li></ul></div></div><div class="col-md-6
footer-link-column"><div
class="footer-link-column-row"><h4>Contribute</h4><ul><li><a
href="//predictionio.apache.org/commu
nity/contribute-code/" target="blank">Contribute</a></li><li><a
href="//github.com/apache/predictionio" target="blank">Source
Code</a></li><li><a href="//issues.apache.org/jira/browse/PIO"
target="blank">Bug Tracker</a></li><li><a
href="mailto:[email protected]"; target="blank">Subscribe to
Development Mailing List</a></li></ul></div></div></div><div class="row"><div
class="col-md-12 footer-link-column"><p>Apache PredictionIO, PredictionIO,
Apache, the Apache feather logo, and the Apache PredictionIO project logo are
either registered trademarks or trademarks of The Apache Software Foundation in
the United States and other countries.</p><p>All other marks mentioned may be
trademarks or registered trademarks of their respective
owners.</p></div></div></div><div id="footer-bottom"><div
class="container"><div class="row"><div class="col-md-12"><div
id="footer-logo-wrapper"><img alt="PredictionIO"
src="/images/logos/logo-white-d1e9c6e6.png"/><span>®</span></div><div
id="social-icons-wrapper"><a class="github-button"
href="https://github.com/apache/predictionio"; data-icon="octicon-star"
data-show-count="true" aria-label="Star apache/predictionio on GitHub">Star</a>
<a class="github-button" href="https://github.com/apache/predictionio/fork";
data-icon="octicon-repo-forked" data-show-count="true" aria-label="Fork
apache/predictionio on GitHub">Fork</a> <script id="github-bjs" async=""
defer="" src="https://buttons.github.io/buttons.js";></script><a
href="https://twitter.com/predictionio"; target="blank"><img alt="PredictionIO
on Twitter" src="/images/icons/twitter-ea9dc152.png"/></a> <a
href="https://www.facebook.com/predictionio"; target="blank"><img
alt="PredictionIO on Facebook" src="/images/icons/facebook-5c57939c.png"/></a>
</div></div></div></div></div></footer></div><script>(function(w,d,t,u,n,s,e){w['SwiftypeObject']=n;w[n]=w[n]||function(){
(w[n].q=w[n].q||[]).push(arguments);};s=d.createElement(t);
e=d.getElementsByTagName(t)[0];s.async=1;s.src=u;e.parentNode.insertBefore(s,e);
})(window,document,'script','//s.swiftypecdn.com/install/v1/st.js','_st');
