http://git-wip-us.apache.org/repos/asf/predictionio-site/blob/6e02047e/system/deploy-cloudformation/index.html
----------------------------------------------------------------------
diff --git a/system/deploy-cloudformation/index.html
b/system/deploy-cloudformation/index.html
index c5071fa..3abe038 100644
--- a/system/deploy-cloudformation/index.html
+++ b/system/deploy-cloudformation/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html><head><title>Deploying with AWS
CloudFormation</title><meta charset="utf-8"/><meta content="IE=edge,chrome=1"
http-equiv="X-UA-Compatible"/><meta name="viewport"
content="width=device-width, initial-scale=1.0"/><meta class="swiftype"
name="title" data-type="string" content="Deploying with AWS
CloudFormation"/><link rel="canonical"
href="https://predictionio.apache.org/system/deploy-cloudformation/"/><link
href="/images/favicon/normal-b330020a.png" rel="shortcut icon"/><link
href="/images/favicon/apple-c0febcf2.png" rel="apple-touch-icon"/><link
href="//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800"
rel="stylesheet"/><link
href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css"
rel="stylesheet"/><link href="/stylesheets/application-eccfc6cb.css"
rel="stylesheet" type="text/css"/><script
src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.2/html5shiv.min.js"></script><script
src
="//cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script><script
src="//use.typekit.net/pqo0itb.js"></script><script>try{Typekit.load({ async:
true });}catch(e){}</script></head><body><div id="global"><header><div
class="container" id="header-wrapper"><div class="row"><div
class="col-sm-12"><div id="logo-wrapper"><span id="drawer-toggle"></span><a
href="#"></a><a href="http://predictionio.apache.org/";><img alt="Apache
PredictionIO" id="logo"
src="/images/logos/logo-ee2b9bb3.png"/></a><span>®</span></div><div
id="menu-wrapper"><div id="pill-wrapper"><a class="pill left"
href="/gallery/template-gallery">TEMPLATES</a> <a class="pill right"
href="//github.com/apache/predictionio/">OPEN SOURCE</a></div></div><img
class="mobile-search-bar-toggler hidden-md hidden-lg"
src="/images/icons/search-glass-704bd4ff.png"/></div></div></div></header><div
id="search-bar-row-wrapper"><div class="container-fluid"
id="search-bar-row"><div class="row"><div class="col-md-9 co
l-sm-11 col-xs-11"><div class="hidden-md hidden-lg"
id="mobile-page-heading-wrapper"><p>PredictionIO Docs</p><h4>Deploying with AWS
CloudFormation</h4></div><h4 class="hidden-sm hidden-xs">PredictionIO
Docs</h4></div><div class="col-md-3 col-sm-1 col-xs-1 hidden-md hidden-lg"><img
id="left-menu-indicator"
src="/images/icons/down-arrow-dfe9f7fe.png"/></div><div class="col-md-3
col-sm-12 col-xs-12 swiftype-wrapper"><div class="swiftype"><form
class="search-form"><img class="search-box-toggler hidden-xs hidden-sm"
src="/images/icons/search-glass-704bd4ff.png"/><div class="search-box"><img
src="/images/icons/search-glass-704bd4ff.png"/><input type="text"
id="st-search-input" class="st-search-input" placeholder="Search
Doc..."/></div><img class="swiftype-row-hider hidden-md hidden-lg"
src="/images/icons/drawer-toggle-active-fcbef12a.png"/></form></div></div><div
class="mobile-left-menu-toggler hidden-md
hidden-lg"></div></div></div></div><div id="page" class="container-fluid"><div
class=
"row"><div id="left-menu-wrapper" class="col-md-3"><nav id="nav-main"><ul><li
class="level-1"><a class="expandible" href="/"><span>Apache PredictionIO®
Documentation</span></a><ul><li class="level-2"><a class="final"
href="/"><span>Welcome to Apache PredictionIO®</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Getting
Started</span></a><ul><li class="level-2"><a class="final"
href="/start/"><span>A Quick Intro</span></a></li><li class="level-2"><a
class="final" href="/install/"><span>Installing Apache
PredictionIO</span></a></li><li class="level-2"><a class="final"
href="/start/download/"><span>Downloading an Engine Template</span></a></li><li
class="level-2"><a class="final" href="/start/deploy/"><span>Deploying Your
First Engine</span></a></li><li class="level-2"><a class="final"
href="/start/customize/"><span>Customizing the
Engine</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Integrating with Your App</span>
</a><ul><li class="level-2"><a class="final" href="/appintegration/"><span>App
Integration Overview</span></a></li><li class="level-2"><a class="expandible"
href="/sdk/"><span>List of SDKs</span></a><ul><li class="level-3"><a
class="final" href="/sdk/java/"><span>Java & Android SDK</span></a></li><li
class="level-3"><a class="final" href="/sdk/php/"><span>PHP
SDK</span></a></li><li class="level-3"><a class="final"
href="/sdk/python/"><span>Python SDK</span></a></li><li class="level-3"><a
class="final" href="/sdk/ruby/"><span>Ruby SDK</span></a></li><li
class="level-3"><a class="final" href="/sdk/community/"><span>Community Powered
SDKs</span></a></li></ul></li></ul></li><li class="level-1"><a
class="expandible" href="#"><span>Deploying an Engine</span></a><ul><li
class="level-2"><a class="final" href="/deploy/"><span>Deploying as a Web
Service</span></a></li><li class="level-2"><a class="final"
href="/batchpredict/"><span>Batch Predictions</span></a></li><li
class="level-2"><a class
="final" href="/deploy/monitoring/"><span>Monitoring Engine</span></a></li><li
class="level-2"><a class="final" href="/deploy/engineparams/"><span>Setting
Engine Parameters</span></a></li><li class="level-2"><a class="final"
href="/deploy/enginevariants/"><span>Deploying Multiple Engine
Variants</span></a></li><li class="level-2"><a class="final"
href="/deploy/plugin/"><span>Engine Server Plugin</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Customizing an
Engine</span></a><ul><li class="level-2"><a class="final"
href="/customize/"><span>Learning DASE</span></a></li><li class="level-2"><a
class="final" href="/customize/dase/"><span>Implement DASE</span></a></li><li
class="level-2"><a class="final"
href="/customize/troubleshooting/"><span>Troubleshooting Engine
Development</span></a></li><li class="level-2"><a class="final"
href="/api/current/#package"><span>Engine Scala
APIs</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#
"><span>Collecting and Analyzing Data</span></a><ul><li class="level-2"><a
class="final" href="/datacollection/"><span>Event Server
Overview</span></a></li><li class="level-2"><a class="final"
href="/datacollection/eventapi/"><span>Collecting Data with
REST/SDKs</span></a></li><li class="level-2"><a class="final"
href="/datacollection/eventmodel/"><span>Events Modeling</span></a></li><li
class="level-2"><a class="final"
href="/datacollection/webhooks/"><span>Unifying Multichannel Data with
Webhooks</span></a></li><li class="level-2"><a class="final"
href="/datacollection/channel/"><span>Channel</span></a></li><li
class="level-2"><a class="final"
href="/datacollection/batchimport/"><span>Importing Data in
Batch</span></a></li><li class="level-2"><a class="final"
href="/datacollection/analytics/"><span>Using Analytics
Tools</span></a></li><li class="level-2"><a class="final"
href="/datacollection/plugin/"><span>Event Server
Plugin</span></a></li></ul></li><li class="level-1"><a class=
"expandible" href="#"><span>Choosing an Algorithm</span></a><ul><li
class="level-2"><a class="final" href="/algorithm/"><span>Built-in Algorithm
Libraries</span></a></li><li class="level-2"><a class="final"
href="/algorithm/switch/"><span>Switching to Another
Algorithm</span></a></li><li class="level-2"><a class="final"
href="/algorithm/multiple/"><span>Combining Multiple
Algorithms</span></a></li><li class="level-2"><a class="final"
href="/algorithm/custom/"><span>Adding Your Own
Algorithms</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Tuning and Evaluation</span></a><ul><li class="level-2"><a
class="final" href="/evaluation/"><span>Overview</span></a></li><li
class="level-2"><a class="final"
href="/evaluation/paramtuning/"><span>Hyperparameter Tuning</span></a></li><li
class="level-2"><a class="final"
href="/evaluation/evaluationdashboard/"><span>Evaluation
Dashboard</span></a></li><li class="level-2"><a class="final"
href="/evaluation/metriccho
ose/"><span>Choosing Evaluation Metrics</span></a></li><li class="level-2"><a
class="final" href="/evaluation/metricbuild/"><span>Building Evaluation
Metrics</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>System Architecture</span></a><ul><li class="level-2"><a
class="final" href="/system/"><span>Architecture Overview</span></a></li><li
class="level-2"><a class="final" href="/system/anotherdatastore/"><span>Using
Another Data Store</span></a></li></ul></li><li class="level-1"><a
class="expandible" href="#"><span>PredictionIO® Official
Templates</span></a><ul><li class="level-2"><a class="final"
href="/templates/"><span>Intro</span></a></li><li class="level-2"><a
class="expandible" href="#"><span>Recommendation</span></a><ul><li
class="level-3"><a class="final"
href="/templates/recommendation/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/dase/"><span>DASE</span></a></li><li class=
"level-3"><a class="final"
href="/templates/recommendation/evaluation/"><span>Evaluation
Explained</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/recommendation/reading-custom-events/"><span>Read Custom
Events</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/customize-data-prep/"><span>Customize Data
Preparator</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/customize-serving/"><span>Customize
Serving</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/training-with-implicit-preference/"><span>Train
with Implicit Preference</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/blacklist-items/"><span>Filter Recommended
Items by Blacklist in Query</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendati
on/batch-evaluator/"><span>Batch Persistable
Evaluator</span></a></li></ul></li><li class="level-2"><a class="expandible"
href="#"><span>E-Commerce Recommendation</span></a><ul><li class="level-3"><a
class="final" href="/templates/ecommercerecommendation/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/ecommercerecommendation/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/ecommercerecommendation/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/ecommercerecommendation/train-with-rate-event/"><span>Train
with Rate Event</span></a></li><li class="level-3"><a class="final"
href="/templates/ecommercerecommendation/adjust-score/"><span>Adjust
Score</span></a></li></ul></li><li class="level-2"><a class="expandible"
href="#"><span>Similar Product</span></a><ul><li class="level-3"><a
class="final" href="/templates/similarproduct/quickstart/"><span>Quick Start</
span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/similarproduct/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/similarproduct/multi-events-multi-algos/"><span>Multiple
Events and Multiple Algorithms</span></a></li><li class="level-3"><a
class="final"
href="/templates/similarproduct/return-item-properties/"><span>Returns Item
Properties</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/train-with-rate-event/"><span>Train with Rate
Event</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/rid-user-set-event/"><span>Get Rid of Events
for Users</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/recommended-user/"><span>Recommend
Users</span></a></li></ul></li><li class="level-2"><a class="expandible"
href="#"><span>Classificati
on</span></a><ul><li class="level-3"><a class="final"
href="/templates/classification/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/classification/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/classification/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/classification/add-algorithm/"><span>Use Alternative
Algorithm</span></a></li><li class="level-3"><a class="final"
href="/templates/classification/reading-custom-properties/"><span>Read Custom
Properties</span></a></li></ul></li></ul></li><li class="level-1"><a
class="expandible" href="#"><span>Engine Template Gallery</span></a><ul><li
class="level-2"><a class="final"
href="/gallery/template-gallery/"><span>Browse</span></a></li><li
class="level-2"><a class="final"
href="/community/submit-template/"><span>Submit your Engine as a
Template</span></a></li></ul></li><li class="level-1"><a class="expa
ndible" href="#"><span>Demo Tutorials</span></a><ul><li class="level-2"><a
class="final" href="/demo/tapster/"><span>Comics Recommendation
Demo</span></a></li><li class="level-2"><a class="final"
href="/demo/community/"><span>Community Contributed Demo</span></a></li><li
class="level-2"><a class="final" href="/demo/textclassification/"><span>Text
Classification Engine Tutorial</span></a></li></ul></li><li class="level-1"><a
class="expandible" href="/community/"><span>Getting Involved</span></a><ul><li
class="level-2"><a class="final"
href="/community/contribute-code/"><span>Contribute Code</span></a></li><li
class="level-2"><a class="final"
href="/community/contribute-documentation/"><span>Contribute
Documentation</span></a></li><li class="level-2"><a class="final"
href="/community/contribute-sdk/"><span>Contribute a SDK</span></a></li><li
class="level-2"><a class="final"
href="/community/contribute-webhook/"><span>Contribute a
Webhook</span></a></li><li class="level-2"><a class="fi
nal" href="/community/projects/"><span>Community
Projects</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Getting Help</span></a><ul><li class="level-2"><a class="final"
href="/resources/faq/"><span>FAQs</span></a></li><li class="level-2"><a
class="final" href="/support/"><span>Support</span></a></li></ul></li><li
class="level-1"><a class="expandible"
href="#"><span>Resources</span></a><ul><li class="level-2"><a class="final"
href="/cli/"><span>Command-line Interface</span></a></li><li class="level-2"><a
class="final" href="/resources/release/"><span>Release
Cadence</span></a></li><li class="level-2"><a class="final"
href="/resources/intellij/"><span>Developing Engines with IntelliJ
IDEA</span></a></li><li class="level-2"><a class="final"
href="/resources/upgrade/"><span>Upgrade Instructions</span></a></li><li
class="level-2"><a class="final"
href="/resources/glossary/"><span>Glossary</span></a></li></ul></li><li
class="level-1"><a class="expandible
" href="#"><span>Apache Software Foundation</span></a><ul><li
class="level-2"><a class="final" href="https://www.apache.org/";><span>Apache
Homepage</span></a></li><li class="level-2"><a class="final"
href="https://www.apache.org/licenses/";><span>License</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/foundation/sponsorship.html";><span>Sponsorship</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/foundation/thanks.html";><span>Thanks</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/security/";><span>Security</span></a></li></ul></li></ul></nav></div><div
class="col-md-9 col-sm-12"><div class="content-header hidden-md
hidden-lg"><div id="page-title"><h1>Deploying with AWS
CloudFormation</h1></div></div><div id="table-of-content-wrapper"><h5>On this
page</h5><aside id="table-of-contents"><ul> <li> <a
href="#overview">Overview</a> </li> <li> <a
href="#step-by-step">Step-by-Step</a> </li> <li> <
a href="#using-the-cluster">Using the Cluster</a> </li> <li> <a
href="#scaling-the-cluster">Scaling the Cluster</a> </li> <li> <a
href="#support-and-pricing">Support and Pricing</a> </li> </ul> </aside><hr/><a
id="edit-page-link"
href="https://github.com/apache/predictionio/tree/livedoc/docs/manual/source/system/deploy-cloudformation.html.md.erb";><img
src="/images/icons/edit-pencil-d6c1bb3d.png"/>Edit this page</a></div><div
class="content-header hidden-sm hidden-xs"><div id="page-title"><h1>Deploying
with AWS CloudFormation</h1></div></div><div class="content"> <h2 id='overview'
class='header-anchors'>Overview</h2><p>You can scale PredictionIO on AWS with
<a href="http://aws.amazon.com/cloudformation/";>CloudFormation</a>. Here we
have defined a PredictionIO CloudFormation stack that you can deploy a
functional, fully distributed PredictionIO cluster in minutes.</p><h3
id='instances' class='header-anchors'>Instances</h3><p>The PredictionIO
CloudFormation stack creates two types of i
nstance: <strong>compute and storage</strong>. By default, the stack will
launch <strong>1 compute Instance and 3 Storage instances</strong>.</p><p>The
compute instance <em>(ComputeInstance)</em> acts as Spark master. You can
launch extra compute instances <em>(ComputeInstanceExtra)</em> by updating the
stack. The storage instances <em>(StorageInstance)</em> form the core of the
HDFS, ZooKeeper quorum, and HBase storage. Extra storage instances
<em>(StorageInstanceExtra)</em> can be added to the cluster by updating the
stack. They cannot be removed once they are spinned up.</p><p>PredictionIO
Event Server will be launched on all storage instances.</p><h3 id='networking'
class='header-anchors'>Networking</h3><p>The stack will automatically create a
VPC and a subnet with an Internet gateway. All cluster instances will be
launched inside this subnet using a single security group that enables all TCP
communications among all instances within the same group. All compute instances
(includ
ing those that are launched after stack creation) will receive public IPs. All
core storage instances will receive public Elastic IPs.</p><h2
id='step-by-step' class='header-anchors'>Step-by-Step</h2><p>First, you need to
have an active Amazon Web Services account with permissions to use the
following services:</p> <ul> <li>Auto Scaling</li> <li>CloudFormation</li>
<li>EC2</li> <li>VPC</li> </ul> <h3 id='subscribe-to-predictionio-cluster'
class='header-anchors'>Subscribe to PredictionIO Cluster</h3><p>Before you can
start using PredictionIO CloudFormation template, you will need to subscribe to
PredictionIO Cluster AMI through AWS Marketplace. To subscribe, go to this <a
href="https://aws.amazon.com/marketplace/pp/B00S74CY0A";>link</a>. Click the
yellow "Continue" button on the next screen.</p><p><img alt="AWS
Marketplace Subscription Page"
src="/images/cloudformation/awsmp-1-609eb919.png"/></p><p>In the next screen,
click the "Manual Launch" tab, then select your
preferred pricing model. Pick your desired version, then click "Accept
Terms".</p><p><img alt="AWS Marketplace Term Page"
src="/images/cloudformation/awsmp-2-7dd613ba.png"/></p><p>Once your
subscription is in place, you may proceed to the next section.</p><h3
id='start-using-cloudformation' class='header-anchors'>Start Using
CloudFormation</h3><p>From your main AWS console, locate CloudFormation and
click on it.</p><p><img alt="CloudFormation on AWS Console"
src="/images/cloudformation/cf-01-13c7f924.png"/></p><p>This will bring you to
the CloudFormation console below.</p><p><img alt="CloudFormation Console"
src="/images/cloudformation/cf-02-61cd8e67.png"/></p><h3
id='select-the-predictionio-cloudformation-stack-template'
class='header-anchors'>Select the PredictionIO CloudFormation Stack
Template</h3><p>From the CloudFormation console, click on the <strong>Create
New Stack</strong> blue button as shown above. This will bring up the
<strong>Select Template</strong> screen
. Name your stack as you like. Within the <em>Template</em> section, choose
<strong>Specify an Amazon S3 template URL</strong>, and put <a
href="https://s3.amazonaws.com/cloudformation.prediction.io/0.12.1/pio.json";>https://s3.amazonaws.com/cloudformation.prediction.io/0.12.1/pio.json</a>
as the value.</p><p><img alt="CloudFormation Stack Template Selection"
src="/images/cloudformation/cf-03-d017fa2d.png"/></p><p>Click
<strong>Next</strong> when you are done.</p><h3 id='specify-stack-parameters'
class='header-anchors'>Specify Stack Parameters</h3><p>The next screen shows
the stack parameters. You must enter your AWS SSH key pair. For the other
parameters, you can change them to meet your needs or simply use the default
values.</p><p><img alt="Stack Parameters"
src="/images/cloudformation/cf-04-6b545f50.png"/></p> <table><thead> <tr>
<th>Parameter</th> <th>Description</th> </tr> </thead><tbody> <tr>
<td>AWS-KeyPair</td> <td>The AWS SSH key pair name that can be used to access
all ins
tances in the cluster.</td> </tr> <tr> <td>AvailabilityZone</td> <td>Specify
the availability zone that the PredictionIO cluster will be launched in. All
instances of the cluster will be launched into the same zone for optimal
network performance between one another.</td> </tr> <tr>
<td>ComputeInstanceType</td> <td>The EC2 instance type of all compute
instances. Memory-optimized EC2 instances are recommended.</td> </tr> <tr>
<td>ComputeInstanceExtra</td> <td>Number of extra compute instances besides the
core compute instance. This can be increased and decreased.</td> </tr> <tr>
<td>StorageInstanceExtra</td> <td>Number of extra storage instances besides
core storage instances. <strong>Never decrease this value or you will risk data
corruption.</strong></td> </tr> <tr> <td>StorageInstanceExtraSize</td> <td>Size
in GB of each extra storage instance. This can be changed when you add an extra
storage instance.</td> </tr> <tr> <td>StorageInstanceExtraVolumeType</td>
<td>The EBS volume typ
e of each extra storage instance. Valid values are <em>standard</em> and
<em>gp2</em>. This can be changed when you add an extra storage instance.</td>
</tr> <tr> <td>StorageInstanceType</td> <td>The EC2 instance type of all
storage instances. General purpose EC2 instances are recommended.</td> </tr>
<tr> <td>StorageInstanceSize</td> <td>Size in GB of each core storage instance.
This cannot be changed once the cluster is launched.</td> </tr> <tr>
<td>StorageInstanceVolumeType</td> <td>The EBS volume type of each core storage
instance. Valid values are <em>standard</em> and <em>gp2</em>. This cannot be
changed once the cluster is launched.</td> </tr> </tbody></table> <p>Click
<strong>Next</strong> when you are done. You will arrive at the
<strong>Options</strong> screen. You can skip this step if you do not have
other options to specify.</p><p>At the <strong>Review</strong> screen, click
<strong>Create</strong> to finish.</p><h2 id='using-the-cluster'
class='header-anchors'>Using the
Cluster</h2><p>You should see the following when the cluster is being created
after the previous step.</p><p><img alt="Stack Creation"
src="/images/cloudformation/cf-05-92ee63c2.png"/></p><p>Once the stack creation
has finished, you can click on <strong>Events</strong> and select
<strong>Outputs</strong> to arrive to the following screen.</p><p><img
alt="Completed Stack"
src="/images/cloudformation/cf-06-b36528fa.png"/></p><p>Take note of
<strong>PIOComputeMasterPublicIp</strong> and
<strong>PIOStorageMasterPublicIp</strong>. We will now access the cluster and
make sure everything is in place.</p><div class="alert-message
warning"><p>Sometimes the stack is created successfully but not all cluster
services would launch due to potential network glitches or system issues within
a cluster instance. In this case, simply delete and create the stack
again.</p></div><h3 id='verify-compute-instances' class='header-anchors'>Verify
Compute Instances</h3><p>SSH to the master compute instance u
sing the <strong>PIOComputeMasterPublicIp</strong>. In this example, let us
assume the IP address be 54.175.145.84, and your private key file be
<strong>yourkey.pem</strong>.</p><div class="highlight shell"><table
style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align:
right"><pre class="lineno">1</pre></td><td class="code"><pre><span class="gp">$
</span>ssh -i yourkey.pem -A -L 8080:localhost:8080 ubuntu@54.175.145.84
+<!DOCTYPE html><html><head><title>Deploying with AWS
CloudFormation</title><meta charset="utf-8"/><meta content="IE=edge,chrome=1"
http-equiv="X-UA-Compatible"/><meta name="viewport"
content="width=device-width, initial-scale=1.0"/><meta class="swiftype"
name="title" data-type="string" content="Deploying with AWS
CloudFormation"/><link rel="canonical"
href="https://predictionio.apache.org/system/deploy-cloudformation/"/><link
href="/images/favicon/normal-b330020a.png" rel="shortcut icon"/><link
href="/images/favicon/apple-c0febcf2.png" rel="apple-touch-icon"/><link
href="//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800"
rel="stylesheet"/><link
href="//maxcdn.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.min.css"
rel="stylesheet"/><link href="/stylesheets/application-eccfc6cb.css"
rel="stylesheet" type="text/css"/><script
src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.2/html5shiv.min.js"></script><script
src
="//cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script><script
src="//use.typekit.net/pqo0itb.js"></script><script>try{Typekit.load({ async:
true });}catch(e){}</script></head><body><div id="global"><header><div
class="container" id="header-wrapper"><div class="row"><div
class="col-sm-12"><div id="logo-wrapper"><span id="drawer-toggle"></span><a
href="#"></a><a href="http://predictionio.apache.org/";><img alt="Apache
PredictionIO" id="logo"
src="/images/logos/logo-ee2b9bb3.png"/></a><span>®</span></div><div
id="menu-wrapper"><div id="pill-wrapper"><a class="pill left"
href="/gallery/template-gallery">TEMPLATES</a> <a class="pill right"
href="//github.com/apache/predictionio/">OPEN SOURCE</a></div></div><img
class="mobile-search-bar-toggler hidden-md hidden-lg"
src="/images/icons/search-glass-704bd4ff.png"/></div></div></div></header><div
id="search-bar-row-wrapper"><div class="container-fluid"
id="search-bar-row"><div class="row"><div class="col-md-9 co
l-sm-11 col-xs-11"><div class="hidden-md hidden-lg"
id="mobile-page-heading-wrapper"><p>PredictionIO Docs</p><h4>Deploying with AWS
CloudFormation</h4></div><h4 class="hidden-sm hidden-xs">PredictionIO
Docs</h4></div><div class="col-md-3 col-sm-1 col-xs-1 hidden-md hidden-lg"><img
id="left-menu-indicator"
src="/images/icons/down-arrow-dfe9f7fe.png"/></div><div class="col-md-3
col-sm-12 col-xs-12 swiftype-wrapper"><div class="swiftype"><form
class="search-form"><img class="search-box-toggler hidden-xs hidden-sm"
src="/images/icons/search-glass-704bd4ff.png"/><div class="search-box"><img
src="/images/icons/search-glass-704bd4ff.png"/><input type="text"
id="st-search-input" class="st-search-input" placeholder="Search
Doc..."/></div><img class="swiftype-row-hider hidden-md hidden-lg"
src="/images/icons/drawer-toggle-active-fcbef12a.png"/></form></div></div><div
class="mobile-left-menu-toggler hidden-md
hidden-lg"></div></div></div></div><div id="page" class="container-fluid"><div
class=
"row"><div id="left-menu-wrapper" class="col-md-3"><nav id="nav-main"><ul><li
class="level-1"><a class="expandible" href="/"><span>Apache PredictionIO®
Documentation</span></a><ul><li class="level-2"><a class="final"
href="/"><span>Welcome to Apache PredictionIO®</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Getting
Started</span></a><ul><li class="level-2"><a class="final"
href="/start/"><span>A Quick Intro</span></a></li><li class="level-2"><a
class="final" href="/install/"><span>Installing Apache
PredictionIO</span></a></li><li class="level-2"><a class="final"
href="/start/download/"><span>Downloading an Engine Template</span></a></li><li
class="level-2"><a class="final" href="/start/deploy/"><span>Deploying Your
First Engine</span></a></li><li class="level-2"><a class="final"
href="/start/customize/"><span>Customizing the
Engine</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Integrating with Your App</span>
</a><ul><li class="level-2"><a class="final" href="/appintegration/"><span>App
Integration Overview</span></a></li><li class="level-2"><a class="expandible"
href="/sdk/"><span>List of SDKs</span></a><ul><li class="level-3"><a
class="final" href="/sdk/java/"><span>Java & Android SDK</span></a></li><li
class="level-3"><a class="final" href="/sdk/php/"><span>PHP
SDK</span></a></li><li class="level-3"><a class="final"
href="/sdk/python/"><span>Python SDK</span></a></li><li class="level-3"><a
class="final" href="/sdk/ruby/"><span>Ruby SDK</span></a></li><li
class="level-3"><a class="final" href="/sdk/community/"><span>Community Powered
SDKs</span></a></li></ul></li></ul></li><li class="level-1"><a
class="expandible" href="#"><span>Deploying an Engine</span></a><ul><li
class="level-2"><a class="final" href="/deploy/"><span>Deploying as a Web
Service</span></a></li><li class="level-2"><a class="final"
href="/batchpredict/"><span>Batch Predictions</span></a></li><li
class="level-2"><a class
="final" href="/deploy/monitoring/"><span>Monitoring Engine</span></a></li><li
class="level-2"><a class="final" href="/deploy/engineparams/"><span>Setting
Engine Parameters</span></a></li><li class="level-2"><a class="final"
href="/deploy/enginevariants/"><span>Deploying Multiple Engine
Variants</span></a></li><li class="level-2"><a class="final"
href="/deploy/plugin/"><span>Engine Server Plugin</span></a></li></ul></li><li
class="level-1"><a class="expandible" href="#"><span>Customizing an
Engine</span></a><ul><li class="level-2"><a class="final"
href="/customize/"><span>Learning DASE</span></a></li><li class="level-2"><a
class="final" href="/customize/dase/"><span>Implement DASE</span></a></li><li
class="level-2"><a class="final"
href="/customize/troubleshooting/"><span>Troubleshooting Engine
Development</span></a></li><li class="level-2"><a class="final"
href="/api/current/#package"><span>Engine Scala
APIs</span></a></li></ul></li><li class="level-1"><a class="expandible" href="#
"><span>Collecting and Analyzing Data</span></a><ul><li class="level-2"><a
class="final" href="/datacollection/"><span>Event Server
Overview</span></a></li><li class="level-2"><a class="final"
href="/datacollection/eventapi/"><span>Collecting Data with
REST/SDKs</span></a></li><li class="level-2"><a class="final"
href="/datacollection/eventmodel/"><span>Events Modeling</span></a></li><li
class="level-2"><a class="final"
href="/datacollection/webhooks/"><span>Unifying Multichannel Data with
Webhooks</span></a></li><li class="level-2"><a class="final"
href="/datacollection/channel/"><span>Channel</span></a></li><li
class="level-2"><a class="final"
href="/datacollection/batchimport/"><span>Importing Data in
Batch</span></a></li><li class="level-2"><a class="final"
href="/datacollection/analytics/"><span>Using Analytics
Tools</span></a></li><li class="level-2"><a class="final"
href="/datacollection/plugin/"><span>Event Server
Plugin</span></a></li></ul></li><li class="level-1"><a class=
"expandible" href="#"><span>Choosing an Algorithm</span></a><ul><li
class="level-2"><a class="final" href="/algorithm/"><span>Built-in Algorithm
Libraries</span></a></li><li class="level-2"><a class="final"
href="/algorithm/switch/"><span>Switching to Another
Algorithm</span></a></li><li class="level-2"><a class="final"
href="/algorithm/multiple/"><span>Combining Multiple
Algorithms</span></a></li><li class="level-2"><a class="final"
href="/algorithm/custom/"><span>Adding Your Own
Algorithms</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Tuning and Evaluation</span></a><ul><li class="level-2"><a
class="final" href="/evaluation/"><span>Overview</span></a></li><li
class="level-2"><a class="final"
href="/evaluation/paramtuning/"><span>Hyperparameter Tuning</span></a></li><li
class="level-2"><a class="final"
href="/evaluation/evaluationdashboard/"><span>Evaluation
Dashboard</span></a></li><li class="level-2"><a class="final"
href="/evaluation/metriccho
ose/"><span>Choosing Evaluation Metrics</span></a></li><li class="level-2"><a
class="final" href="/evaluation/metricbuild/"><span>Building Evaluation
Metrics</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>System Architecture</span></a><ul><li class="level-2"><a
class="final" href="/system/"><span>Architecture Overview</span></a></li><li
class="level-2"><a class="final" href="/system/anotherdatastore/"><span>Using
Another Data Store</span></a></li></ul></li><li class="level-1"><a
class="expandible" href="#"><span>PredictionIO® Official
Templates</span></a><ul><li class="level-2"><a class="final"
href="/templates/"><span>Intro</span></a></li><li class="level-2"><a
class="expandible" href="#"><span>Recommendation</span></a><ul><li
class="level-3"><a class="final"
href="/templates/recommendation/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/dase/"><span>DASE</span></a></li><li class=
"level-3"><a class="final"
href="/templates/recommendation/evaluation/"><span>Evaluation
Explained</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/recommendation/reading-custom-events/"><span>Read Custom
Events</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/customize-data-prep/"><span>Customize Data
Preparator</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/customize-serving/"><span>Customize
Serving</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/training-with-implicit-preference/"><span>Train
with Implicit Preference</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendation/blacklist-items/"><span>Filter Recommended
Items by Blacklist in Query</span></a></li><li class="level-3"><a class="final"
href="/templates/recommendati
on/batch-evaluator/"><span>Batch Persistable
Evaluator</span></a></li></ul></li><li class="level-2"><a class="expandible"
href="#"><span>E-Commerce Recommendation</span></a><ul><li class="level-3"><a
class="final" href="/templates/ecommercerecommendation/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/ecommercerecommendation/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/ecommercerecommendation/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/ecommercerecommendation/train-with-rate-event/"><span>Train
with Rate Event</span></a></li><li class="level-3"><a class="final"
href="/templates/ecommercerecommendation/adjust-score/"><span>Adjust
Score</span></a></li></ul></li><li class="level-2"><a class="expandible"
href="#"><span>Similar Product</span></a><ul><li class="level-3"><a
class="final" href="/templates/similarproduct/quickstart/"><span>Quick Start</
span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/similarproduct/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/similarproduct/multi-events-multi-algos/"><span>Multiple
Events and Multiple Algorithms</span></a></li><li class="level-3"><a
class="final"
href="/templates/similarproduct/return-item-properties/"><span>Returns Item
Properties</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/train-with-rate-event/"><span>Train with Rate
Event</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/rid-user-set-event/"><span>Get Rid of Events
for Users</span></a></li><li class="level-3"><a class="final"
href="/templates/similarproduct/recommended-user/"><span>Recommend
Users</span></a></li></ul></li><li class="level-2"><a class="expandible"
href="#"><span>Classificati
on</span></a><ul><li class="level-3"><a class="final"
href="/templates/classification/quickstart/"><span>Quick
Start</span></a></li><li class="level-3"><a class="final"
href="/templates/classification/dase/"><span>DASE</span></a></li><li
class="level-3"><a class="final"
href="/templates/classification/how-to/"><span>How-To</span></a></li><li
class="level-3"><a class="final"
href="/templates/classification/add-algorithm/"><span>Use Alternative
Algorithm</span></a></li><li class="level-3"><a class="final"
href="/templates/classification/reading-custom-properties/"><span>Read Custom
Properties</span></a></li></ul></li></ul></li><li class="level-1"><a
class="expandible" href="#"><span>Engine Template Gallery</span></a><ul><li
class="level-2"><a class="final"
href="/gallery/template-gallery/"><span>Browse</span></a></li><li
class="level-2"><a class="final"
href="/community/submit-template/"><span>Submit your Engine as a
Template</span></a></li></ul></li><li class="level-1"><a class="expa
ndible" href="#"><span>Demo Tutorials</span></a><ul><li class="level-2"><a
class="final" href="/demo/tapster/"><span>Comics Recommendation
Demo</span></a></li><li class="level-2"><a class="final"
href="/demo/community/"><span>Community Contributed Demo</span></a></li><li
class="level-2"><a class="final" href="/demo/textclassification/"><span>Text
Classification Engine Tutorial</span></a></li></ul></li><li class="level-1"><a
class="expandible" href="/community/"><span>Getting Involved</span></a><ul><li
class="level-2"><a class="final"
href="/community/contribute-code/"><span>Contribute Code</span></a></li><li
class="level-2"><a class="final"
href="/community/contribute-documentation/"><span>Contribute
Documentation</span></a></li><li class="level-2"><a class="final"
href="/community/contribute-sdk/"><span>Contribute a SDK</span></a></li><li
class="level-2"><a class="final"
href="/community/contribute-webhook/"><span>Contribute a
Webhook</span></a></li><li class="level-2"><a class="fi
nal" href="/community/projects/"><span>Community
Projects</span></a></li></ul></li><li class="level-1"><a class="expandible"
href="#"><span>Getting Help</span></a><ul><li class="level-2"><a class="final"
href="/resources/faq/"><span>FAQs</span></a></li><li class="level-2"><a
class="final" href="/support/"><span>Support</span></a></li></ul></li><li
class="level-1"><a class="expandible"
href="#"><span>Resources</span></a><ul><li class="level-2"><a class="final"
href="/cli/"><span>Command-line Interface</span></a></li><li class="level-2"><a
class="final" href="/resources/release/"><span>Release
Cadence</span></a></li><li class="level-2"><a class="final"
href="/resources/intellij/"><span>Developing Engines with IntelliJ
IDEA</span></a></li><li class="level-2"><a class="final"
href="/resources/upgrade/"><span>Upgrade Instructions</span></a></li><li
class="level-2"><a class="final"
href="/resources/glossary/"><span>Glossary</span></a></li></ul></li><li
class="level-1"><a class="expandible
" href="#"><span>Apache Software Foundation</span></a><ul><li
class="level-2"><a class="final" href="https://www.apache.org/";><span>Apache
Homepage</span></a></li><li class="level-2"><a class="final"
href="https://www.apache.org/licenses/";><span>License</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/foundation/sponsorship.html";><span>Sponsorship</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/foundation/thanks.html";><span>Thanks</span></a></li><li
class="level-2"><a class="final"
href="https://www.apache.org/security/";><span>Security</span></a></li></ul></li></ul></nav></div><div
class="col-md-9 col-sm-12"><div class="content-header hidden-md
hidden-lg"><div id="page-title"><h1>Deploying with AWS
CloudFormation</h1></div></div><div id="table-of-content-wrapper"><h5>On this
page</h5><aside id="table-of-contents"><ul> <li> <a
href="#overview">Overview</a> </li> <li> <a
href="#step-by-step">Step-by-Step</a> </li> <li> <
a href="#using-the-cluster">Using the Cluster</a> </li> <li> <a
href="#scaling-the-cluster">Scaling the Cluster</a> </li> <li> <a
href="#support-and-pricing">Support and Pricing</a> </li> </ul> </aside><hr/><a
id="edit-page-link"
href="https://github.com/apache/predictionio/tree/livedoc/docs/manual/source/system/deploy-cloudformation.html.md.erb";><img
src="/images/icons/edit-pencil-d6c1bb3d.png"/>Edit this page</a></div><div
class="content-header hidden-sm hidden-xs"><div id="page-title"><h1>Deploying
with AWS CloudFormation</h1></div></div><div class="content"> <h2 id='overview'
class='header-anchors'>Overview</h2><p>You can scale PredictionIO on AWS with
<a href="http://aws.amazon.com/cloudformation/";>CloudFormation</a>. Here we
have defined a PredictionIO CloudFormation stack that you can deploy a
functional, fully distributed PredictionIO cluster in minutes.</p><h3
id='instances' class='header-anchors'>Instances</h3><p>The PredictionIO
CloudFormation stack creates two types of i
nstance: <strong>compute and storage</strong>. By default, the stack will
launch <strong>1 compute Instance and 3 Storage instances</strong>.</p><p>The
compute instance <em>(ComputeInstance)</em> acts as Spark master. You can
launch extra compute instances <em>(ComputeInstanceExtra)</em> by updating the
stack. The storage instances <em>(StorageInstance)</em> form the core of the
HDFS, ZooKeeper quorum, and HBase storage. Extra storage instances
<em>(StorageInstanceExtra)</em> can be added to the cluster by updating the
stack. They cannot be removed once they are spinned up.</p><p>PredictionIO
Event Server will be launched on all storage instances.</p><h3 id='networking'
class='header-anchors'>Networking</h3><p>The stack will automatically create a
VPC and a subnet with an Internet gateway. All cluster instances will be
launched inside this subnet using a single security group that enables all TCP
communications among all instances within the same group. All compute instances
(includ
ing those that are launched after stack creation) will receive public IPs. All
core storage instances will receive public Elastic IPs.</p><h2
id='step-by-step' class='header-anchors'>Step-by-Step</h2><p>First, you need to
have an active Amazon Web Services account with permissions to use the
following services:</p> <ul> <li>Auto Scaling</li> <li>CloudFormation</li>
<li>EC2</li> <li>VPC</li> </ul> <h3 id='subscribe-to-predictionio-cluster'
class='header-anchors'>Subscribe to PredictionIO Cluster</h3><p>Before you can
start using PredictionIO CloudFormation template, you will need to subscribe to
PredictionIO Cluster AMI through AWS Marketplace. To subscribe, go to this <a
href="https://aws.amazon.com/marketplace/pp/B00S74CY0A";>link</a>. Click the
yellow "Continue" button on the next screen.</p><p><img alt="AWS
Marketplace Subscription Page"
src="/images/cloudformation/awsmp-1-609eb919.png"/></p><p>In the next screen,
click the "Manual Launch" tab, then select your
preferred pricing model. Pick your desired version, then click "Accept
Terms".</p><p><img alt="AWS Marketplace Term Page"
src="/images/cloudformation/awsmp-2-7dd613ba.png"/></p><p>Once your
subscription is in place, you may proceed to the next section.</p><h3
id='start-using-cloudformation' class='header-anchors'>Start Using
CloudFormation</h3><p>From your main AWS console, locate CloudFormation and
click on it.</p><p><img alt="CloudFormation on AWS Console"
src="/images/cloudformation/cf-01-13c7f924.png"/></p><p>This will bring you to
the CloudFormation console below.</p><p><img alt="CloudFormation Console"
src="/images/cloudformation/cf-02-61cd8e67.png"/></p><h3
id='select-the-predictionio-cloudformation-stack-template'
class='header-anchors'>Select the PredictionIO CloudFormation Stack
Template</h3><p>From the CloudFormation console, click on the <strong>Create
New Stack</strong> blue button as shown above. This will bring up the
<strong>Select Template</strong> screen
. Name your stack as you like. Within the <em>Template</em> section, choose
<strong>Specify an Amazon S3 template URL</strong>, and put <a
href="https://s3.amazonaws.com/cloudformation.prediction.io/0.13.0/pio.json";>https://s3.amazonaws.com/cloudformation.prediction.io/0.13.0/pio.json</a>
as the value.</p><p><img alt="CloudFormation Stack Template Selection"
src="/images/cloudformation/cf-03-d017fa2d.png"/></p><p>Click
<strong>Next</strong> when you are done.</p><h3 id='specify-stack-parameters'
class='header-anchors'>Specify Stack Parameters</h3><p>The next screen shows
the stack parameters. You must enter your AWS SSH key pair. For the other
parameters, you can change them to meet your needs or simply use the default
values.</p><p><img alt="Stack Parameters"
src="/images/cloudformation/cf-04-6b545f50.png"/></p> <table><thead> <tr>
<th>Parameter</th> <th>Description</th> </tr> </thead><tbody> <tr>
<td>AWS-KeyPair</td> <td>The AWS SSH key pair name that can be used to access
all ins
tances in the cluster.</td> </tr> <tr> <td>AvailabilityZone</td> <td>Specify
the availability zone that the PredictionIO cluster will be launched in. All
instances of the cluster will be launched into the same zone for optimal
network performance between one another.</td> </tr> <tr>
<td>ComputeInstanceType</td> <td>The EC2 instance type of all compute
instances. Memory-optimized EC2 instances are recommended.</td> </tr> <tr>
<td>ComputeInstanceExtra</td> <td>Number of extra compute instances besides the
core compute instance. This can be increased and decreased.</td> </tr> <tr>
<td>StorageInstanceExtra</td> <td>Number of extra storage instances besides
core storage instances. <strong>Never decrease this value or you will risk data
corruption.</strong></td> </tr> <tr> <td>StorageInstanceExtraSize</td> <td>Size
in GB of each extra storage instance. This can be changed when you add an extra
storage instance.</td> </tr> <tr> <td>StorageInstanceExtraVolumeType</td>
<td>The EBS volume typ
e of each extra storage instance. Valid values are <em>standard</em> and
<em>gp2</em>. This can be changed when you add an extra storage instance.</td>
</tr> <tr> <td>StorageInstanceType</td> <td>The EC2 instance type of all
storage instances. General purpose EC2 instances are recommended.</td> </tr>
<tr> <td>StorageInstanceSize</td> <td>Size in GB of each core storage instance.
This cannot be changed once the cluster is launched.</td> </tr> <tr>
<td>StorageInstanceVolumeType</td> <td>The EBS volume type of each core storage
instance. Valid values are <em>standard</em> and <em>gp2</em>. This cannot be
changed once the cluster is launched.</td> </tr> </tbody></table> <p>Click
<strong>Next</strong> when you are done. You will arrive at the
<strong>Options</strong> screen. You can skip this step if you do not have
other options to specify.</p><p>At the <strong>Review</strong> screen, click
<strong>Create</strong> to finish.</p><h2 id='using-the-cluster'
class='header-anchors'>Using the
Cluster</h2><p>You should see the following when the cluster is being created
after the previous step.</p><p><img alt="Stack Creation"
src="/images/cloudformation/cf-05-92ee63c2.png"/></p><p>Once the stack creation
has finished, you can click on <strong>Events</strong> and select
<strong>Outputs</strong> to arrive to the following screen.</p><p><img
alt="Completed Stack"
src="/images/cloudformation/cf-06-b36528fa.png"/></p><p>Take note of
<strong>PIOComputeMasterPublicIp</strong> and
<strong>PIOStorageMasterPublicIp</strong>. We will now access the cluster and
make sure everything is in place.</p><div class="alert-message
warning"><p>Sometimes the stack is created successfully but not all cluster
services would launch due to potential network glitches or system issues within
a cluster instance. In this case, simply delete and create the stack
again.</p></div><h3 id='verify-compute-instances' class='header-anchors'>Verify
Compute Instances</h3><p>SSH to the master compute instance u
sing the <strong>PIOComputeMasterPublicIp</strong>. In this example, let us
assume the IP address be 54.175.145.84, and your private key file be
<strong>yourkey.pem</strong>.</p><div class="highlight shell"><table
style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align:
right"><pre class="lineno">1</pre></td><td class="code"><pre><span class="gp">$
</span>ssh -i yourkey.pem -A -L 8080:localhost:8080 ubuntu@54.175.145.84
</pre></td></tr></tbody></table> </div> <p>Once you are in, point your web
browser to <a href="http://localhost:8080";>http://localhost:8080</a>. You
should see something similar to the following.</p><p><img alt="Example Spark
UI" src="/images/cloudformation/spark-019c2cf1.png"/></p><div
class="alert-message note"><p>In the example above
<strong>NumberOfComputeWorkers</strong> is <strong>2</strong>. This is because
the example has 1 compute instance and 1 extra compute instance. If you do not
have any extra compute instances, you will see only 1 worker on the above
page.</p></div><h3 id='verify-storage-instances' class='header-anchors'>Verify
Storage Instances</h3><p>SSH to the storage instance using the
<strong>PIOStorageMasterPublicIp</strong>. In this example, let us assume the
IP address be 54.175.1.36, and your private key file be
<strong>yourkey.pem</strong>.</p><div class="highlight shell"><table
style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align:
rig
ht"><pre class="lineno">1</pre></td><td class="code"><pre><span class="gp">$
</span>ssh -i yourkey.pem -A -L 50070:localhost:50070 -L 16010:localhost:16010
-L 16030:localhost:16030 ubuntu@54.175.1.36
</pre></td></tr></tbody></table> </div> <p>Once you are in, point your web
browser to <a href="http://localhost:50070";>http://localhost:50070</a> and
click on <strong>Datainstances</strong> at the top menu. You should see the
following page.</p><p><img alt="Example HDFS UI"
src="/images/cloudformation/hdfs-51f0231f.png"/></p><div class="alert-message
warning"><p>All <strong>3 storage instances</strong> must be up for proper
operation.</p></div><p>If all <strong>3 storage instances</strong> are working
properly, you can then verify HBase by pointing your web browser to <a
href="http://localhost:16010";>http://localhost:16010</a>. You should see
something similar to the following.</p><p><img alt="Example HBase UI"
src="/images/cloudformation/hbase-32538c47.png"/></p><p>If you do not specify
any extra storage instances, you should see 2 region servers. There should also
be 1 backup master.</p><h3 id='running-quick-start'
class='header-anchors'>Running Quick Start</h3><p>You can now star
t with the fully-distributed PredictionIO cluster. Let's start with the <a
href="/templates/recommendation/quickstart/">recommendation quick start</a>
with a few twists.</p> <ol> <li><p>Skip the installation steps and run
<code>pio status</code>. You should see everything functional.</p></li>
<li><p>Run through the section <strong>Create a Sample App</strong> as
described. The installation directory of PredictionIO is
<code>/opt/PredictionIO</code>.</p></li> <li><p>Run through the section
<strong>Collecting Data</strong> as described, except that you will be
connecting to the Event Server at the master core storage instance. Assuming
the private IP of the master core storage instance is <code>10.0.0.123</code>,
add <code>--url http://10.0.0.123:7070</code> to the
<code>import_eventserver.py</code> command.</p></li> <li><p>Copy HBase
configuration to the engine template directory. The full path of the
configuration file is <code>/opt/hbase-0.98.9-hadoop2/conf/hbase-site.xml</code
>. (This step will not be required in future releases.)</p></li> <li><p>Run
>through the section <strong>Deploy the Engine as a Service</strong> up to the
>subsection <strong>Training</strong>. Assuming the private DNS name of the
>master compute instance is <code>ip-10-0-0-234.ec2.internal</code>, add
><code>-- --master spark://ip-10-0-0-234.ec2.internal:7077</code> after the
><code>pio train</code> command. This will send the training to the compute
>cluster instead of the local machine. The Spark master URL must match exactly
>the one shown on its web UI. Repeat the same steps for subsection
><strong>Deploying</strong>, which will create an Engine Server backed by the
>compute cluster.</p></li> </ol> <h2 id='scaling-the-cluster'
>class='header-anchors'>Scaling the Cluster</h2><p>As your data size and/or
>audience grow, you can scale your cluster to handle more workload or decrease
>turnaround time. In this section, we will provide some general guidelines
>about when and how to scale your clus
ter with CloudFormation.</p><h3 id='scaling-compute-instances'
class='header-anchors'>Scaling Compute Instances</h3><p>You can increase
compute instances to reduce training time <em>($pio train)</em> and the time to
query an engine server. You can also check the [Spark Master Web UI] to see if
you need additional compute power.</p><p>Notice that for compute instances, you
can increase or decrease the number of extra compute instances
<em>(ComputeInstanceExtra)</em> as much as you like. The extra compute
instances will join the master and become slave compute instances as Spark
workers.</p><p>Let us begin by adding 2 extra compute instances. At the
CloudFormation console, right click on the cluster stack and click on
<strong>Update Stack</strong>.</p><p><img alt="Updating Stack to Add Extra
Compute Instances"
src="/images/cloudformation/compute-1-c572ea84.png"/></p><p>At the
<strong>Select Template</strong> screen, make sure <strong>Use existing
template</strong> is picked, then clic
k <strong>Next</strong>.</p><p>At the <strong>Specify Parameters</strong>
screen, increase the value of <strong>ComputeInstanceExtra</strong> to
<strong>2</strong>, then click <strong>Next</strong>.</p><p><img alt="Adding 2
Extra Compute Instances"
src="/images/cloudformation/compute-2-f7da8e72.png"/></p><p>At the
<strong>Options</strong> screen, leave everything unchanged, and click
<strong>Next</strong>.</p><p>At the <strong>Review</strong> screen, make sure
<strong>ComputeInstanceExtra</strong> is now updated to <strong>2</strong>.
Finish by clicking <strong>Update</strong>.</p><p><img alt="Review Changes of
Adding Extra Compute Instances"
src="/images/cloudformation/compute-3-ee5fc6a8.png"/></p><p>You will be brought
back to the CloudFormation console. You should see the stack status changed to
<strong>UPDATE_IN_PROGRESS</strong>.</p><p><img alt="Adding Extra Compute
Instances In Progress"
src="/images/cloudformation/compute-4-67fe0caa.png"/></p><p>Once the status
become <strong
>UPDATE_COMPLETED</strong>, you will have 2 extra compute instances. Notice
>that during the update, your cluster is still functional and any existing
>work will not be affected. If you are downscaling, existing work might be
>affected during the update process.</p><h3 id='scaling-storage-instances'
>class='header-anchors'>Scaling Storage Instances</h3><p>You can scale your
>storage instances when you are about to run out of space. You can check your
>storage usage at the [Hadoop NameNode web UI].</p><div class="alert-message
>warning"><p>For storage instances, you can only increase the number of extra
>storage instances <em>(StorageInstanceExtra)</em> within the bounds of AWS
>EC2 limits. Decreasing the instances will risk data
>corruption.</p></div><p>Let us begin by adding 2 extra storage instances. At
>the CloudFormation console, right click on the cluster stack and click on
><strong>Update Stack</strong>.</p><p><img alt="Updating Stack to Add Extra
>Storage Instances" src="/images/cloudform
ation/compute-1-c572ea84.png"/></p><p>At the <strong>Select Template</strong>
screen, make sure <strong>Use existing template</strong> is picked, then click
<strong>Next</strong>.</p><p>At the <strong>Specify Parameters</strong> screen,
increase the value of <strong>StorageInstanceExtra</strong> to
<strong>2</strong>, and set the value of
<strong>StorageInstanceExtraSize</strong> to <strong>100</strong>, then click
<strong>Next</strong>. Notice that whenever you add an extra storage instance,
you can change its size to a new value. The new size will not affect existing
storage instances and your data will be safe.</p><p><img alt="Adding 2 Extra
Storage Instances"
src="/images/cloudformation/storage-1-ca3ae3a6.png"/></p><p>At the
<strong>Options</strong> screen, leave everything unchanged, and click
<strong>Next</strong>.</p><p>At the <strong>Review</strong> screen, make sure
<strong>StorageInstanceExtra</strong> is now updated to <strong>2</strong>, and
<strong>StorageInstanceExtraS
ize</strong> is updated to <strong>100</strong>. Finish by clicking
<strong>Update</strong>.</p><p><img alt="Review Changes of Adding Extra Storage
Instances" src="/images/cloudformation/storage-2-9395c9f2.png"/></p><p>You will
be brought back to the CloudFormation console. You should see the stack status
changed to <strong>UPDATE_IN_PROGRESS</strong>.</p><p><img alt="Adding Extra
Compute Instances In Progress"
src="/images/cloudformation/compute-4-67fe0caa.png"/></p><p>Once the status
become <strong>UPDATE_COMPLETED</strong>, you will have 2 extra storage
instances. Notice that during the up-scaling update, your cluster is still
functional and existing work will not be affected. They may be affected during
downscale.</p><h2 id='support-and-pricing' class='header-anchors'>Support and
Pricing</h2><p>PredictionIO Cluster comes with Enterprise Support. For pricing
and support details, please contact <a
href="mailto:supp...@prediction.io";>supp...@prediction.io</a>.</p></div></div></div>
</div><footer><div class="container"><div class="seperator"></div><div
class="row"><div class="col-md-6 footer-link-column"><div
class="footer-link-column-row"><h4>Community</h4><ul><li><a
href="//predictionio.apache.org/install/"
target="blank">Download</a></li><li><a href="//predictionio.apache.org/"
target="blank">Docs</a></li><li><a href="//github.com/apache/predictionio"
target="blank">GitHub</a></li><li><a
href="mailto:user-subscr...@predictionio.apache.org"; target="blank">Subscribe
to User Mailing List</a></li><li><a
href="//stackoverflow.com/questions/tagged/predictionio"
target="blank">Stackoverflow</a></li></ul></div></div><div class="col-md-6
footer-link-column"><div
class="footer-link-column-row"><h4>Contribute</h4><ul><li><a
href="//predictionio.apache.org/community/contribute-code/"
target="blank">Contribute</a></li><li><a
href="//github.com/apache/predictionio" target="blank">Source
Code</a></li><li><a href="//issues.apache.org/jira/browse/PIO"
target="blank">Bug Trac
ker</a></li><li><a href="mailto:dev-subscr...@predictionio.apache.org";
target="blank">Subscribe to Development Mailing
List</a></li></ul></div></div></div><div class="row"><div class="col-md-12
footer-link-column"><p>Apache PredictionIO, PredictionIO, Apache, the Apache
feather logo, and the Apache PredictionIO project logo are either registered
trademarks or trademarks of The Apache Software Foundation in the United States
and other countries.</p><p>All other marks mentioned may be trademarks or
registered trademarks of their respective owners.</p></div></div></div><div
id="footer-bottom"><div class="container"><div class="row"><div
class="col-md-12"><div id="footer-logo-wrapper"><img alt="PredictionIO"
src="/images/logos/logo-white-d1e9c6e6.png"/><span>®</span></div><div
id="social-icons-wrapper"><a class="github-button"
href="https://github.com/apache/predictionio"; data-icon="octicon-star"
data-show-count="true" aria-label="Star apache/predictionio on GitHub">Star</a>
<a class="
github-button" href="https://github.com/apache/predictionio/fork";
data-icon="octicon-repo-forked" data-show-count="true" aria-label="Fork
apache/predictionio on GitHub">Fork</a> <script id="github-bjs" async=""
defer="" src="https://buttons.github.io/buttons.js";></script><a
href="https://twitter.com/predictionio"; target="blank"><img alt="PredictionIO
on Twitter" src="/images/icons/twitter-ea9dc152.png"/></a> <a
href="https://www.facebook.com/predictionio"; target="blank"><img
alt="PredictionIO on Facebook" src="/images/icons/facebook-5c57939c.png"/></a>
</div></div></div></div></div></footer></div><script>(function(w,d,t,u,n,s,e){w['SwiftypeObject']=n;w[n]=w[n]||function(){
(w[n].q=w[n].q||[]).push(arguments);};s=d.createElement(t);
