Added: websites/staging/slider/trunk/content/docs/configuration/resource_specification.html ============================================================================== --- websites/staging/slider/trunk/content/docs/configuration/resource_specification.html (added) +++ websites/staging/slider/trunk/content/docs/configuration/resource_specification.html Wed Mar 25 17:59:38 2015 @@ -0,0 +1,814 @@ +<!DOCTYPE html> +<html lang="en"> +<head> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE- 2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <link href="/css/bootstrap.min.css" rel="stylesheet"> + <link href="/css/bootstrap-theme.min.css" rel="stylesheet"> + <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js";></script> + <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js";></script> + <![endif]--> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js";></script> + <script src="/js/bootstrap.min.js"></script> + <link href="/css/slider.css" rel="stylesheet" type="text/css"> + <link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet"> + <title></title> + +</head> + +<body> +<nav class="navbar navbar-default navbar-fixed-top" role="navigation"> +<div class="container-fluid"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar-items"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="/index.html">Slider</a> + </div> + <div class="collapse navbar-collapse" id="navbar-items"> + <ul class="nav navbar-nav"> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#"> + Project <span class="caret"></span> + </a> +<ul class="dropdown-menu"> +<li id="nav_index"><a href="/index.html">Home</a></li> +<li id="nav_downloads"><a href="/downloads">Downloads</a></li> +<li id="nav_source"><a href="https://git-wip-us.apache.org/repos/asf/incubator-slider.git";><i class="fa fa-external-link"></i> Source</a></li> +<li><a href="http://www.apache.org/licenses/LICENSE-2.0";><i class="fa fa-external-link"></i> License</a></li> +<li class="divider"></li> +<li class="dropdown-header">Community</li> +<li id="nav_team"><a href="/team.html">Team</a></li> +<li id="nav_mailinglists"><a href="/mailing_lists.html">Mailing Lists</a></li> +<li class="divider"></li> +<li class="dropdown-header">Tools</li> +<li id="nav_jira"><a href="https://issues.apache.org/jira/browse/SLIDER";><i class="fa fa-external-link"></i> Issues</a></li> +<li id="nav_builds"><a href="https://builds.apache.org/job/Slider-develop";><i class="fa fa-external-link"></i> Builds</a></li> +</ul> +</li> + + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="/design"> + Design <span class="caret"></span> + </a> +<ul class="dropdown-menu"> +<li id="nav_design_overview"><a href="/design/architecture.html">Overview</a></li> +<li id="nav_app_needs"><a href="/docs/slider_specs/application_needs.html">Application Needs</a></li> +<li id="nav_spec"><a href="/design/specification/index.html">Expected Behavior</a></li> +<li id="nav_registry"><a href="/design/registry/index.html">Service Registry</a></li> +<li id="nav_ssl_impl"><a href="/design/ssl_implementation.html">SSL Implementation</a></li> +<li id="nav_role_history"><a href="/design/rolehistory.html">Role History</a></li> +</ul> +</li> + + + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="/developing"> + Developing <span class="caret"></span> + </a> +<ul class="dropdown-menu"> +<li id="nav_dev_building"><a href="/developing/building.html">Building</a></li> +<li id="nav_dev_debugging"><a href="/developing/debugging.html">Debugging</a></li> +<li id="nav_dev_testing"><a href="/developing/testing.html">Testing</a></li> +<li id="nav_dev_funtesting"><a href="/developing/functional_tests.html">Functional Testing</a></li> +<li id="nav_dev_manual_testing"><a href="/developing/manual_testing.html">Manual Testing</a></li> +<li id="nav_releasing"><a href="/developing/releasing.html">Releasing</a></li> +</ul> +</li> + + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#"> + Using <span class="caret"></span> + </a> +<ul class="dropdown-menu"> +<li id="nav_use_getting_started"><a href="/docs/getting_started.html">Getting Started</a></li> +<li id="nav_use_manpage"><a href="/docs/manpage.html">Man Page</a></li> +<li role="presentation" class="divider"></li> +<li id="nav_use_app_package"><a href="/docs/slider_specs/index.html">Creating App Packages</a></li> +<li role="presentation" class="divider"></li> +<li id="nav_use_client_config"><a href="/docs/client-configuration.html">Client Configuration</a></li> +<li id="nav_use_exitcodes"><a href="/docs/exitcodes.html">Client Exit Codes</a></li> +<li role="presentation" class="divider"></li> +<li id="nav_use_debugging"><a href="/developing/debugging.html">Debugging</a></li> +<li id="nav_use_troubleshooting"><a href="/docs/troubleshooting.html">Troubleshooting</a></li> +<li role="presentation" class="divider"></li> +<li id="nav_use_high_availability"><a href="/docs/high_availability.html">High Availability</a></li> +<li id="nav_use_security"><a href="/docs/security.html">Security</a></li> +<li id="nav_am_agent_ssl"><a href="/docs/ssl.html">Agent to AM SSL</a></li> +<li id="nav_use_app_configuration"><a href="/docs/configuration/index.html">App Configuration Model</a></li> +<li id="nav_use_examples"><a href="/docs/examples.html">Provider Examples</a></li> +</ul> +</li> + + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#"> + ASF Links <span class="caret"></span> + </a> +<ul class="dropdown-menu"> +<li><a href="http://www.apache.org";><i class="fa fa-external-link"></i> Apache Software Foundation</a></li> +<li><a href="http://www.apache.org/foundation/sponsorship.html";><i class="fa fa-external-link"></i> Sponsorship</a></li> +<li><a href="http://www.apache.org/security/";><i class="fa fa-external-link"></i> Security</a></li> +<li><a href="http://www.apache.org/foundation/thanks.html";><i class="fa fa-external-link"></i> Thanks</a></li> +</ul> +</li> +</ul> + + </div> + +</div> +</nav> + + +<div class="container-fluid"> +<div class="row"> + <div class="col-md-2" id="sidebar"> + <div style="text-align: center"> + <h1><a href="/index.html">Apache Slider (incubating)</a></h1> + <hr> +Latest release: <strong>0.60.0-incubating</strong><br> + <br> + <a id="download-button-sidebar" class="btn btn-success btn-block" href="/downloads/" role="button">Download</a> + </div> + <hr> + <a href="http://incubator.apache.org";><img id="logo" alt="Apache Incubator ™" class="img-responsive" src="http://incubator.apache.org/images/apache-incubator-logo.png"/></a> + </div> + <div class="col-md-8 col-md-offset-1"> + + <div id="content"> + + <h1 class="title"></h1> + + <!--- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<h1 id="apache-slider-resource-specification">Apache Slider Resource Specification</h1> +<ul> +<li><a href="#core">Core Properties</a></li> +<li><a href="#failurepolicy">Container Failure Policy</a></li> +<li><a href="#placement">Placement Policies and escalation</a></li> +<li><a href="#labels">Labels</a></li> +<li><a href="#logagg">Log Aggregation</a></li> +</ul> +<p>The Resource specification file, <code>resources.json</code>, defines the Yarn resource needs for each component type that belong to the application. +This includes: +<em> container CPU and memory requirements +</em> component placement policy, including YARN labels to explictly request nodes on. +<em> failure policy: what to do if components keep failing. +</em> placement escalation policy +* where logs generated by applications will be saved, information which is passed to YARN to enable +these logs to be copied to HDFS and remotely retrieved, even while the application is running</p> +<p>As such, it is the core file used by Slider to configure and manage +the application.</p> +<h2 id="wzxhzdk14wzxhzdk15core-properties"><a name="core"></a>Core Properties</h2> +<p>An example resource requirement for an application that has two components "master" and "worker" is as follows. Slider will automatically add the requirements for the AppMaster for the application. This component is named "slider-appmaster".</p> +<p>Some parameters that can be specified for a component instance include:</p> +<table> + <tr> + <td>yarn.component.instances</td> + <td> + Number of instances of this component type + </td> + </tr> + <tr> + <td>yarn.memory</td> + <td> + Amount of memory in MB required for the component instance. + </td> + </tr> + <tr> + <td>yarn.vcores</td> + <td> + Number of "virtual cores" requested + </td> + </tr> + <tr> + <td>yarn.role.priority</td> + <td> + Unique priority for this component + </td> + </tr> + +</table> + +<h3 id="component-instance-count-yarncomponentinstances">Component instance count: <code>yarn.component.instances</code></h3> +<p>The property <code>yarn.component.instances</code> is one of the most foundational one in slider +âit declares how many instances of a component to instantiate on the cluster.</p> +<p>If the value is set to "0", no instances of a component will be created. If set +to a larger number, more instances will be requested. Thus the property sets the size +of the application, component-by-component.</p> +<p>The number of instances of each component is application-specific; there are no recommended +values.</p> +<h3 id="container-resource-requirements-yarnmemory-and-yarnvcores">Container resource requirements: <code>yarn.memory</code> and <code>yarn.vcores</code></h3> +<p>These two properties define how much memory and CPU capacity each +YARN container of this component requires. YARN will queue +container requests until enough capacity exists within the cluster +to satisfy them. When there is capacity, a container is allocated to Slider, +which then deploys an instance of the component. </p> +<p>The larger these numbers, the more capacity the application gets.</p> +<p>If more memory or CPU is requested than needed then that containers will take +longer to be allocated than necessary, and other work may not be scheduled: +the cluster will be under-utilized.</p> +<p><code>yarn.memory</code> declares the amount of memory to ask for in YARN containers; it should +be defined for each component based on the expected memory consumption. It is measured +in MB.</p> +<p>If the cluster has hard memory limits enabled, then if the processes in a container +use more physical or virtual memory than was granted âYARN will kill the container. +Slider will attempt to recreate the component instance by requesting a new container, +though if the number of failures of a component is too great then it will eventually +give up and fail the application.</p> +<p>A YARN cluster is usually configured with a minimum container allocation, set in <code>yarn-site.xml</code> +by the configuration parameter <code>yarn.scheduler.minimum-allocation-mb</code>; the default value is +1024 MB. It will also have a maximum size set in <code>yarn.scheduler.maximum-allocation-mb</code>; +the default is 8192, that is, 8GB. Asking for more than this will result in the request +being rejected.</p> +<p><code>yarn.vcores</code> declares the number of "virtual cores" to request. These are a site-configured +fraction of a physical CPU core; if the ratio of virtual to physical is 1:1 then a physical core +is allocated to each one (this may include a Hyperthreaded Core if enabled in the BIOS). +If the ratio is lower, such as 2:1, then each vcore allocates half a physical one.</p> +<p>This notion of a virtual core is intended to partially isolate applications from differences +in cluster performance: a process which needs 2 vcores on one cluster should ideally +still ask for 2 vcores on a different cluster âeven if the latter has newer CPU parts. +In practise, it's not so consistent. Ask for more vcores if your process needs more CPU +time. </p> +<p>YARN clusters may be configured to throttle CPU usage: if a process tries to use more than +has been granted to the container, it will simply be scheduled with less CPU time. The penalty +for using more CPU than requested is therefore less destructive than attempting to +use more memory than requested/granted.</p> +<h4 id="relationship-between-yarnmemory-and-jvm-heap-size">Relationship between <code>yarn.memory</code> and JVM Heap Size</h4> +<p>Java applications deployed by Slider usually have a JVM heap size property which needs +to be defined as part of the application configuration.</p> +<p>The value of <code>yarn.memory</code> MUST be bigger than the heap size allocated to any JVM, as a JVM +uses a lot more memory than simply the heap alone. We have found that asking for at least 50% +more appears to work, though some experimentation will be needed. </p> +<p>Slider does not attempt to derive a heap size for any component from the YARN allocation.</p> +<h3 id="component-instance-count-yarnrolepriority">Component instance count: <code>yarn.role.priority</code></h3> +<p>The property <code>yarn.role.priority</code> has two purposes within Slider +1. It provides a unique index of individual component types. That is, it is not +the name of a component which Slider uses to index components, it is it's priority +value. +1. It defines the priority within an application for YARN to use when allocating +components. Components with higher priority get allocated first.</p> +<p>Generally the latter use, YARN allocation priority, is less important for Slider-deployed +applications than for analytics applications designed to scale to as many nodes that can +be instantiated. A static slider cluster has a predefined number of of each components to +request (defined by <code>yarn.component.instances</code>), with memory and CPU requirements of +each component's container defined by <code>yarn.memory</code> and <code>yarn.vcores</code>. It will request +the specified number of components âand keep those requests outstanding until they are +satisfied.</p> +<h3 id="example">Example</h3> +<div class="codehilite"><pre><span class="p">{</span> + "<span class="n">schema</span>" <span class="p">:</span> "<span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">example</span><span class="p">.</span><span class="n">org</span><span class="o">/</span><span class="n">specification</span><span class="o">/</span><span class="n">v2</span><span class="p">.</span>0<span class="p">.</span>0"<span class="p">,</span> + "<span class="n">metadata</span>" <span class="p">:</span> <span class="p">{</span> + <span class="p">},</span> + "<span class="k">global</span>" <span class="p">:</span> <span class="p">{</span> + + <span class="p">},</span> + "<span class="n">components</span>" <span class="p">:</span> <span class="p">{</span> + "<span class="n">HBASE_MASTER</span>" <span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>" <span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>" <span class="p">:</span> "1" + "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>" <span class="p">:</span> "768"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>" <span class="p">:</span> "1" + <span class="p">},</span> + "<span class="n">slider</span><span class="o">-</span><span class="n">appmaster</span>" <span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>" <span class="p">:</span> "1024"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>" <span class="p">:</span> "1" + <span class="p">},</span> + "<span class="n">HBASE_REGIONSERVER</span>" <span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>" <span class="p">:</span> "2"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>" <span class="p">:</span> "1" + <span class="p">}</span> + <span class="p">}</span> +<span class="p">}</span> +</pre></div> + + +<h2 id="wzxhzdk16wzxhzdk17the-slider-appmaster-component"><a name="slider-appmaster"></a>The <code>slider-appmaster</code> component</h2> +<p>The examples here all have a component <code>slider-appmaster</code>. This defines the settings of +the application master itself: the memory and CPU it requires, optionally a label (see +<a href="#labels">"Labels"</a>). The <code>yarn.role.priority</code> value is ignored: the priority is always "0"; +and the instance count, <code>yarn.component.instances</code> is implicitly set to "1". </p> +<p>The entry exists primarily to allow applications to configure the amount of RAM the AM should +request.</p> +<h2 id="wzxhzdk18wzxhzdk19container-failure-policy"><a name="failurepolicy"></a>Container Failure Policy</h2> +<p>YARN containers hosting component instances may fail. This can happen because of</p> +<ol> +<li>A problem in the configuration of the instance.</li> +<li>A problem in the app package</li> +<li>A problem (hardware, software or networking) in the server hosting the container</li> +<li>Conflict for a resource (usually a network port) between the component instance +and another running program.</li> +<li>The server or the network connection to it failing.</li> +<li>The server being taken down for maintenance.</li> +</ol> +<p>Slider reacts to a failed container by requesting a new container from YARN, +preferably on a host that has already hosted an instance of that role. Once +the container is allocated, slider will redeploy an instance of the component. +As it may time for YARN to have the resources to allocate the container, it +may take some time for the replacement to be instantiated.</p> +<p>Slider tracks failures in an attempt to differentiate problems in the +application package or its configuration from those of the underlying servers. +If a a component fails <em>too many times</em> then slider considers the application +itself as failing, and halts.</p> +<p>This leads to the question: what is too many times?</p> +<p>The limits are defined in <code>resources.json</code>: +1. The duration of a failure window, a time period in which failures are counted. +This duration can span days. +1. The maximum number of failures of any component in this time period.</p> +<h3 id="failure-threshold-for-a-component">Failure threshold for a component</h3> +<p>The number of times a component may fail within a failure window is +defined by the property <code>yarn.container.failure.threshold</code></p> +<p>If set to "0" there are no limits on +the number of times containers may fail.</p> +<p>The failure thresholds for individual components can be set independently</p> +<h3 id="failure-window">Failure window</h3> +<p>The failure window can be set by minutes, days and hours. These must be set +in the <code>global</code> options, as they apply to slider only.</p> +<div class="codehilite"><pre><span class="n">yarn</span><span class="p">.</span><span class="n">container</span><span class="p">.</span><span class="n">failure</span><span class="p">.</span><span class="n">window</span><span class="p">.</span><span class="n">days</span> +<span class="n">yarn</span><span class="p">.</span><span class="n">container</span><span class="p">.</span><span class="n">failure</span><span class="p">.</span><span class="n">window</span><span class="p">.</span><span class="n">hours</span> +<span class="n">yarn</span><span class="p">.</span><span class="n">container</span><span class="p">.</span><span class="n">failure</span><span class="p">.</span><span class="n">window</span><span class="p">.</span><span class="n">minutes</span> +</pre></div> + + +<p>These properties define the duration of the window; they are all combined +so the window is, in minutes:</p> +<div class="codehilite"><pre><span class="n">minutes</span> <span class="o">+</span> 60 <span class="o">*</span> <span class="p">(</span><span class="n">hours</span> <span class="o">+</span> <span class="n">days</span> <span class="o">*</span> 24<span class="p">)</span> +</pre></div> + + +<p>The initial window is measured from the +start of the application master âonce the duration of that window +is exceeded, all failure counts are reset, and the window begins again.</p> +<p>If the AM itself fails, the failure counts are reset and and the window is +restarted.</p> +<p>The default value is <code>yarn.container.failure.window.hours=6</code>; when changing +the window size, the hour value must be explicitly set, even if to zero, to +change this.</p> +<h3 id="recommended-values">Recommended values</h3> +<p>We recommend having a duration of a few hours for the window, and a +large failure limit proportional to the the number of instances of that +component </p> +<p>Why? </p> +<p>This will cover the loss of a large portion of the hardware of the cluster by +trying to reinstantiate all the components. Yet, if a component does fail +repeatedly, eventually slider will conclude that there is a problem and fail +with the exit code 73, <code>EXIT_DEPLOYMENT_FAILED</code>. </p> +<h3 id="example_1">Example</h3> +<p>Here is a <code>resource.json</code> file for an HBase cluster:</p> +<div class="codehilite"><pre><span class="p">{</span> + "<span class="n">schema</span>" <span class="p">:</span> "<span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">example</span><span class="p">.</span><span class="n">org</span><span class="o">/</span><span class="n">specification</span><span class="o">/</span><span class="n">v2</span><span class="p">.</span>0<span class="p">.</span>0"<span class="p">,</span> + "<span class="n">metadata</span>" <span class="p">:</span> <span class="p">{</span> <span class="p">},</span> + "<span class="k">global</span>" <span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">container</span><span class="p">.</span><span class="n">failure</span><span class="p">.</span><span class="n">threshold</span>" <span class="p">:</span> "4"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">container</span><span class="p">.</span><span class="n">failure</span><span class="p">.</span><span class="n">window</span><span class="p">.</span><span class="n">days</span>" <span class="p">:</span> "0<span class="o">'</span><span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">container</span><span class="p">.</span><span class="n">failure</span><span class="p">.</span><span class="n">window</span><span class="p">.</span><span class="n">hours</span>" <span class="p">:</span> "1<span class="o">'</span><span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">container</span><span class="p">.</span><span class="n">failure</span><span class="p">.</span><span class="n">window</span><span class="p">.</span><span class="n">minutes</span>" <span class="p">:</span> "0<span class="o">'</span> + <span class="p">},</span> + "<span class="n">components</span>" <span class="p">:</span> <span class="p">{</span> + "<span class="n">slider</span><span class="o">-</span><span class="n">appmaster</span>" <span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>" <span class="p">:</span> "256"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>" <span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>" <span class="p">:</span> "1" + <span class="p">},</span> + "<span class="n">HBASE_MASTER</span>" <span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>" <span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>" <span class="p">:</span> "256"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>" <span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>" <span class="p">:</span> "2" + <span class="p">},</span> + "<span class="n">HBASE_REGIONSERVER</span>" <span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>" <span class="p">:</span> "2"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>" <span class="p">:</span> "512"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">container</span><span class="p">.</span><span class="n">failure</span><span class="p">.</span><span class="n">threshold</span>" <span class="p">:</span> "15"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>" <span class="p">:</span> "2"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>" <span class="p">:</span> "10" + <span class="p">}</span> + <span class="p">}</span> +<span class="p">}</span> +</pre></div> + + +<p>The window size is set to one hour: after that the counters are reset.</p> +<p>There is a global failure threshold of 4 components.</p> +<p>There are ten worker components requested; the failure threshold for these +components is overridden to be fifteen. Given that there are more region servers +than masters, a higher failure rate of worker nodes is to be expected <strong>if the cause of +the failure is due to the underlying hardware</strong></p> +<p>Choosing a higher value for the region servers ensures that the application is resilient +to harware problems. If there were some configuration problem in the region server +deployments, resulting in them all failing rapidly, this threshold would soon be breached +which would cause the application to fail. Thus, configuration problems would be detected.</p> +<p>These failure thresholds are all heuristics. When initially configuring an +application instance, low thresholds reduce the disruption caused by components +which are frequently failing due to configuration problems. </p> +<p>In a production application, large failure thresholds and/or shorter windows +ensures that the application is resilient to transient failures of the underlying +YARN cluster and hardware.</p> +<h2 id="wzxhzdk20wzxhzdk21placement-policies-and-escalation"><a name="placement"></a>Placement Policies and escalation</h2> +<p>Slider can be configured with different options for <strong>placement</strong> âthe +policies by which it chooses where to ask YARN for nodes.</p> +<h3 id="placement-policy">Placement Policy</h3> +<p>The "placement policy" of a component is the set of rules by which Slider makes +a decision on where to request instances of that component from YARN.</p> +<table> +<tr> + <td>0</td> + <td> + Default: placement is spread + across the cluster on re-starts, with escalation if requests are + unmet. Unreliable nodes are avoided. + </td> +</tr> + +<tr> + <td>1</td> + <td>strict: a component is requested on every node used, irrespective + of faiure history. No escalation takes place.</td> +</tr> + +<tr> + <td>2</td> + <td>Anywhere. Place requests anywhere and ignore the history.</td> +</tr> + +<tr> + <td>4</td> + <td>Anti affinity required. This option is not currently supported.</td> +</tr> + +</table> + +<p>The placement policy is a binary "or" of all the values, and can be +set in the property <code>"yarn.component.placement.policy"</code>.</p> +<p>Example:</p> +<div class="codehilite"><pre>"<span class="n">HBASE_REST</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>"<span class="p">:</span> "3"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">placement</span><span class="p">.</span><span class="n">policy</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>"<span class="p">:</span> "556" +<span class="p">},</span> +</pre></div> + + +<p>This defines an HBASE_REST component with a placement policy of "1"; strict.</p> +<p>On application restarts Slider will re-request the same node.</p> +<p>If the component were configured to request an explicit port for its REST endpoint, +then the same URL would reach it whenever this application were deployed +âprovided the host was available and the port not already in use.</p> +<h4 id="notes">Notes</h4> +<ol> +<li> +<p>There's no support for <strong>anti-affinity</strong> âi.e. to mandate that component +instances must never be deployed on the same hosts. Once YARN adds support for +this, Slider will support it.</p> +</li> +<li> +<p>Slider never explicitly black-lists nodes. It does track which nodes have been +unreliable "recently", and avoids explicitly requesting them. If YARN does +actually allocate a container there, Slider will attempt to deploy the component +there.</p> +</li> +<li> +<p>Apart from an (optional) label, placement policies for the application master itself + cannot be specified. The Application Master is deployed wherever YARN sees fit.</p> +</li> +</ol> +<h3 id="node-failure-threshold-yarnnodefailurethreshold">Node Failure Threshold, <code>yarn.node.failure.threshold</code></h3> +<p>The configuration property <code>yarn.node.failure.threshold</code> defines how "unreliable" +a node must be before it is skipped for placement requests.</p> +<ol> +<li>This is per-component.</li> +<li>It is ignored for "strict" or "anywhere" placements.</li> +<li>It is reset at the same time as the container failure counters; that is, at +the interval defined by the <code>yarn.container.failure.window</code> properties</li> +</ol> +<h3 id="escalation-yarnplacementescalateseconds">Escalation: <code>yarn.placement.escalate.seconds</code></h3> +<p>For any component whose placement policy is not "any", Slider saves to HDFS +a record the nodes on which instances were running. When starting a cluster, it uses +this history to identify hosts on which to request instances. </p> +<ol> +<li>Slider initially asks for nodes on those specific hosts âprovided their recent failure +history is considered acceptable.</li> +<li>It tracks which 'placed' requests are outstanding.</li> +<li>If, after the specified escalation time, YARN containers have not been allocated +on those nodes, slider will "escalate" the placement of those requests that are +outstanding.</li> +<li>It currently does this by cancelling each request and re-requesting a container on +that node, this time with the <code>relaxLocality</code> flag set.</li> +<li>This tells YARN to seek an alternative location in the cluster if it cannot +allocate one on the target host.</li> +<li>If there is enough capacity in the cluster, the new node will then be allocated.</li> +</ol> +<p>The higher the cost of migrating a component instance from one host to another, the longer +we would recommend for an escalation timeout.</p> +<p>Example:</p> +<div class="codehilite"><pre><span class="p">{</span> + "<span class="n">schema</span>"<span class="p">:</span> "<span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">example</span><span class="p">.</span><span class="n">org</span><span class="o">/</span><span class="n">specification</span><span class="o">/</span><span class="n">v2</span><span class="p">.</span>0<span class="p">.</span>0"<span class="p">,</span> + "<span class="n">metadata</span>"<span class="p">:</span> <span class="p">{</span> + <span class="p">},</span> + "<span class="k">global</span>"<span class="p">:</span> <span class="p">{</span> + <span class="p">},</span> + "<span class="n">components</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">HBASE_MASTER</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">placement</span><span class="p">.</span><span class="n">escalate</span><span class="p">.</span><span class="n">seconds</span>"<span class="p">:</span> "10" + <span class="p">},</span> + "<span class="n">HBASE_REGIONSERVER</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>"<span class="p">:</span> "2"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "10"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">placement</span><span class="p">.</span><span class="n">escalate</span><span class="p">.</span><span class="n">seconds</span>"<span class="p">:</span> "600" + <span class="p">},</span> + "<span class="n">slider</span><span class="o">-</span><span class="n">appmaster</span>"<span class="p">:</span> <span class="p">{</span> + <span class="p">}</span> + <span class="p">}</span> +<span class="p">}</span> +</pre></div> + + +<p>This declares that the <code>HBASE_MASTER</code> placement should be escalated after one second, +but that that <code>HBASE_REGIONSERVER</code> instances should have an escalation timeout of 600 +seconds âten minutes. These values were chosen as an HBase Master can be allocated +anywhere in the cluster, but a region server is significantly faster if restarted +on the same node on which it previously saved all its data. Even though HDFS will +have replicated all data elsewhere, it will have been scattered across the cluster +âresulting in remote access for most of the data, at least until a full compaction +has taken place.</p> +<h4 id="notes_1">Notes</h4> +<ol> +<li> +<p>Escalation goes directory from "specific node" to "anywhere in cluster"; it does +not have any intermediate "same-rack" policy.</p> +</li> +<li> +<p>If components were assigned to specific labels, then even when placement is +"escalated", Slider will always ask for containers on the specified labels. That +is âit will never relax the constraint of "deploy on the labels specified". If +there are not enough labelled nodes for the application, either the cluster +administrators need to add more labelled nodes, or the application must be reconfigured +with a different label policy.</p> +</li> +<li> +<p>Escalated components may be allocated containers on nodes which already have a running +instance of the same component.</p> +</li> +<li> +<p>If the placement policy is "strict", there is no escalation. If the node +is not available or lacks capacity, the request will remain unsatisfied.</p> +</li> +<li> +<p>There is no placement escalation option for the application master.</p> +</li> +<li> +<p>For more details, see: <a href="/design/rolehistory.html">Role History</a></p> +</li> +</ol> +<h2 id="wzxhzdk22wzxhzdk23using-labels"><a name="labels"></a>Using Labels</h2> +<p>The <code>resources.json</code> file include specifications the labels to be used when allocating containers for the components. The details of the YARN Label feature can be found at <a href="https://issues.apache.org/jira/browse/YARN-796";>YARN-796</a>.</p> +<p>In summary: </p> +<ul> +<li>Nodes can be assigned one or more labels</li> +<li>Capacity Queues can be defined with access to one or more labels</li> +<li>Ensure application components are associated with appropriate label expressions </li> +<li>Create the application using specific queue</li> +</ul> +<p>This way, you can guarantee that a certain set of nodes are reserved for an application or for a component within an application.</p> +<p>Label expression is specified through property <code>yarn.label.expression</code>. When no label expression is specified then it is assumed that only non-labeled nodes are used when allocating containers for component instances.</p> +<p>If a label expression is specified for the <code>slider-appmaster</code> component then it also becomes the default label expression for all component. </p> +<h4 id="example_2">Example</h4> +<p>Here is a <code>resource.json</code> file for an HBase cluster which uses labels.</p> +<p>The label for the application master is <code>hbase1</code> and the label expression for the HBASE_MASTER components is <code>hbase1_master</code>. +<code>HBASE_REGIONSERVER</code> instances will automatically use label <code>hbase1</code>.</p> +<div class="codehilite"><pre><span class="p">{</span> + "<span class="n">schema</span>"<span class="p">:</span> "<span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">example</span><span class="p">.</span><span class="n">org</span><span class="o">/</span><span class="n">specification</span><span class="o">/</span><span class="n">v2</span><span class="p">.</span>0<span class="p">.</span>0"<span class="p">,</span> + "<span class="n">metadata</span>"<span class="p">:</span> <span class="p">{</span> + <span class="p">},</span> + "<span class="k">global</span>"<span class="p">:</span> <span class="p">{</span> + <span class="p">},</span> +"<span class="n">components</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">HBASE_MASTER</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">label</span><span class="p">.</span><span class="n">expression</span>"<span class="p">:</span>"<span class="n">hbase1_master</span>" + <span class="p">},</span> + "<span class="n">HBASE_REGIONSERVER</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>"<span class="p">:</span> "2"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "10"<span class="p">,</span> + <span class="p">},</span> + "<span class="n">slider</span><span class="o">-</span><span class="n">appmaster</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">label</span><span class="p">.</span><span class="n">expression</span>"<span class="p">:</span>"<span class="n">hbase1</span>" + <span class="p">}</span> + <span class="p">}</span> +<span class="p">}</span> +</pre></div> + + +<p>To deploy this application in a YARN cluster, the following steps must be followed.</p> +<ol> +<li>Create two labels, <code>hbase1</code> and <code>hbase1_master</code> (use <code>yarn rmadmin</code> commands) </li> +<li>Assign the labels to nodes (use <code>yarn rmadmin</code> commands) </li> +<li>Perform refresh queue (<code>yarn -refreshqueue</code>) </li> +<li>Create a queue by defining it in the capacity scheduler configuragion.</li> +<li>Allow the queue to access to the labels and ensure that appropriate min/max capacity is assigned </li> +<li>Perform refresh queue (<code>yarn -refreshqueue</code>) </li> +<li>Create the Slider application against the above queue using parameter --queue while creating the application</li> +</ol> +<h3 id="notes_2">Notes</h3> +<ol> +<li> +<p>If a label is defined in the <code>global</code> section, it will also apply to all components which do +not explicitly identify a label. If such a label is expression is set there and another is defined +for the <code>slider-appmaster</code>, the app master's label is only used for its placement.</p> +</li> +<li> +<p>To explicitly request that components are not requested on a label, irrespective of +any global- or appmaster- spettings, set the <code>yarn.label.expression</code> to an empty string:</p> +<div class="codehilite"><pre>"<span class="n">HBASE_REGIONSERVER</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>"<span class="p">:</span> "2"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "10"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">label</span><span class="p">.</span><span class="n">expression</span>"<span class="p">:</span>"" +<span class="p">}</span> +</pre></div> + + +</li> +<li> +<p>If there is not enough capacity within a set of labelled nodes for the requested containers, +the application instance will not reach its requested size.</p> +</li> +</ol> +<h2 id="wzxhzdk24wzxhzdk25log-aggregation"><a name="logagg"></a>Log Aggregation</h2> +<p>Log aggregation at regular intervals for long running services (LRS) needs to be enabled at the YARN level before +any application can exploit this functionality. To enable set the following property to a positive value of 3600 (in secs) +or more. If set to a positive value less than 3600 (1 hour) this property defaults to 3600. To disable log aggregation +set it to -1.</p> +<div class="codehilite"><pre> <span class="nt"><property></span> + <span class="nt"><name></span>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds<span class="nt"></name></span> + <span class="nt"><value></span>3600<span class="nt"></value></span> + <span class="nt"></property></span> +</pre></div> + + +<p>Subsequently every application owner has the flexibility to set the include and exclude patterns of file names that +they intent to aggregate. In Slider the resources.json file can be used to specify the include and exclude patterns +of files that need to be backed up under the default log directory of the application. These properties needs to be +set at the global level as shown below -</p> +<div class="codehilite"><pre><span class="p">{</span> + "<span class="n">schema</span>"<span class="p">:</span> "<span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">example</span><span class="p">.</span><span class="n">org</span><span class="o">/</span><span class="n">specification</span><span class="o">/</span><span class="n">v2</span><span class="p">.</span>0<span class="p">.</span>0"<span class="p">,</span> + "<span class="n">metadata</span>"<span class="p">:</span> <span class="p">{</span> + <span class="p">},</span> + "<span class="k">global</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="nb">log</span><span class="p">.</span><span class="n">include</span><span class="p">.</span><span class="n">patterns</span>"<span class="p">:</span> "<span class="n">hbase</span><span class="o">*.*</span>"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="nb">log</span><span class="p">.</span><span class="n">exclude</span><span class="p">.</span><span class="n">patterns</span>"<span class="p">:</span> "<span class="n">hbase</span><span class="o">*</span><span class="p">.</span><span class="n">out</span>" + <span class="p">},</span> + "<span class="n">components</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">HBASE_MASTER</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "1"<span class="p">,</span> + <span class="p">},</span> + "<span class="n">HBASE_REGIONSERVER</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>"<span class="p">:</span> "2"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "10"<span class="p">,</span> + <span class="p">},</span> + "<span class="n">slider</span><span class="o">-</span><span class="n">appmaster</span>"<span class="p">:</span> <span class="p">{</span> + <span class="p">}</span> + <span class="p">}</span> +<span class="p">}</span> +</pre></div> + + +<p>The details of the YARN Log Aggregation feature can be found at <a href="https://issues.apache.org/jira/browse/YARN-2468";>YARN-2468</a>.</p> +<h2 id="putting-it-all-together">Putting it all together</h2> +<p>Here is an example of a definition of an HBase cluster.</p> +<div class="codehilite"><pre><span class="p">{</span> + "<span class="n">schema</span>"<span class="p">:</span> "<span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">example</span><span class="p">.</span><span class="n">org</span><span class="o">/</span><span class="n">specification</span><span class="o">/</span><span class="n">v2</span><span class="p">.</span>0<span class="p">.</span>0"<span class="p">,</span> + "<span class="n">metadata</span>"<span class="p">:</span> <span class="p">{</span> + <span class="p">},</span> + "<span class="k">global</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="nb">log</span><span class="p">.</span><span class="n">include</span><span class="p">.</span><span class="n">patterns</span>"<span class="p">:</span> "<span class="n">hbase</span><span class="o">*.*</span>"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="nb">log</span><span class="p">.</span><span class="n">exclude</span><span class="p">.</span><span class="n">patterns</span>"<span class="p">:</span> "<span class="n">hbase</span><span class="o">*</span><span class="p">.</span><span class="n">out</span>"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">container</span><span class="p">.</span><span class="n">failure</span><span class="p">.</span><span class="n">window</span><span class="p">.</span><span class="n">hours</span>"<span class="p">:</span> "0"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">container</span><span class="p">.</span><span class="n">failure</span><span class="p">.</span><span class="n">window</span><span class="p">.</span><span class="n">minutes</span>"<span class="p">:</span> "30"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">label</span><span class="p">.</span><span class="n">expression</span>"<span class="p">:</span>"<span class="n">development</span>" + <span class="p">},</span> + "<span class="n">components</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">slider</span><span class="o">-</span><span class="n">appmaster</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>"<span class="p">:</span> "1024"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>"<span class="p">:</span> "1" + "<span class="n">yarn</span><span class="p">.</span><span class="n">label</span><span class="p">.</span><span class="n">expression</span>"<span class="p">:</span>"" + <span class="p">},</span> + "<span class="n">HBASE_MASTER</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">placement</span><span class="p">.</span><span class="n">escalate</span><span class="p">.</span><span class="n">seconds</span>"<span class="p">:</span> "10"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>"<span class="p">:</span> "1500" + <span class="p">},</span> + "<span class="n">HBASE_REGIONSERVER</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>"<span class="p">:</span> "2"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>"<span class="p">:</span> "1500"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">container</span><span class="p">.</span><span class="n">failure</span><span class="p">.</span><span class="n">threshold</span>"<span class="p">:</span> "15"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">placement</span><span class="p">.</span><span class="n">escalate</span><span class="p">.</span><span class="n">seconds</span>"<span class="p">:</span> "60" + <span class="p">},</span> + "<span class="n">HBASE_REST</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>"<span class="p">:</span> "3"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">placement</span><span class="p">.</span><span class="n">policy</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">container</span><span class="p">.</span><span class="n">failure</span><span class="p">.</span><span class="n">threshold</span>"<span class="p">:</span> "3"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>"<span class="p">:</span> "556" + <span class="p">},</span> + "<span class="n">HBASE_THRIFT</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>"<span class="p">:</span> "4"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "0"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">placement</span><span class="p">.</span><span class="n">policy</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>"<span class="p">:</span> "556" + "<span class="n">yarn</span><span class="p">.</span><span class="n">label</span><span class="p">.</span><span class="n">expression</span>"<span class="p">:</span>"<span class="n">stable</span>" + <span class="p">},</span> + "<span class="n">HBASE_THRIFT2</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">role</span><span class="p">.</span><span class="n">priority</span>"<span class="p">:</span> "5"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">component</span><span class="p">.</span><span class="n">placement</span><span class="p">.</span><span class="n">policy</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>"<span class="p">:</span> "556" + "<span class="n">yarn</span><span class="p">.</span><span class="n">label</span><span class="p">.</span><span class="n">expression</span>"<span class="p">:</span>"<span class="n">stable</span>" + <span class="p">}</span> + <span class="p">}</span> +<span class="p">}</span> +</pre></div> + + +<p>There are ten region servers, with a 60-second timeout for placement escalation; +15 containers can fail in the "recent" time window before the application is +considered to have failed.</p> +<p>The time window to reset failures is set to 30 minutes.</p> +<p>The Thrift, Thrift2 and REST servers all have strict placement. The REST +server also has a container failure threshold of 3: if it can not come up +three times, the entire application deployment is considered a failure.</p> +<p>The default label for nodes is "development". For the application master itself it is "", +meaning anywhere. Both thrift services are requested on the labels "stable"</p> + </div> + + <div id="footer"> + <a alt="Apache Software Foundation" href="http://www.apache.org";> + <img id="asf-logo" alt="Apache Software Foundation" src="/images/feather-small.gif" width="100"> + </a> + <div class="copyright"> + <p> + Copyright © 2011-2014 The Apache Software Foundation, Licensed under + the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>. + Apache Slider, Slider, Apache, the Apache feather logo, and the Apache Incubator + project logo are trademarks of the <a href="http://www.apache.org";>Apache Software Foundation</a>.<br /> + Site created with <a href="http://getbootstrap.com/";>Bootstrap</a> including icons from <a href="http://glyphicons.com/";>GLYPHICONS</a> and <a href="http://fontawesome.io/";>Font Awesome</a>. + </p> + </div> + </div> + </div> + +</div> +</div> + +<script type="text/javascript"> + +</script> +</body> +</html>
Added: websites/staging/slider/trunk/content/docs/configuration/revision-1/index.html ============================================================================== --- websites/staging/slider/trunk/content/docs/configuration/revision-1/index.html (added) +++ websites/staging/slider/trunk/content/docs/configuration/revision-1/index.html Wed Mar 25 17:59:38 2015 @@ -0,0 +1,219 @@ +<!DOCTYPE html> +<html lang="en"> +<head> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE- 2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <link href="/css/bootstrap.min.css" rel="stylesheet"> + <link href="/css/bootstrap-theme.min.css" rel="stylesheet"> + <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js";></script> + <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js";></script> + <![endif]--> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js";></script> + <script src="/js/bootstrap.min.js"></script> + <link href="/css/slider.css" rel="stylesheet" type="text/css"> + <link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet"> + <title></title> + +</head> + +<body> +<nav class="navbar navbar-default navbar-fixed-top" role="navigation"> +<div class="container-fluid"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar-items"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="/index.html">Slider</a> + </div> + <div class="collapse navbar-collapse" id="navbar-items"> + <ul class="nav navbar-nav"> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#"> + Project <span class="caret"></span> + </a> +<ul class="dropdown-menu"> +<li id="nav_index"><a href="/index.html">Home</a></li> +<li id="nav_downloads"><a href="/downloads">Downloads</a></li> +<li id="nav_source"><a href="https://git-wip-us.apache.org/repos/asf/incubator-slider.git";><i class="fa fa-external-link"></i> Source</a></li> +<li><a href="http://www.apache.org/licenses/LICENSE-2.0";><i class="fa fa-external-link"></i> License</a></li> +<li class="divider"></li> +<li class="dropdown-header">Community</li> +<li id="nav_team"><a href="/team.html">Team</a></li> +<li id="nav_mailinglists"><a href="/mailing_lists.html">Mailing Lists</a></li> +<li class="divider"></li> +<li class="dropdown-header">Tools</li> +<li id="nav_jira"><a href="https://issues.apache.org/jira/browse/SLIDER";><i class="fa fa-external-link"></i> Issues</a></li> +<li id="nav_builds"><a href="https://builds.apache.org/job/Slider-develop";><i class="fa fa-external-link"></i> Builds</a></li> +</ul> +</li> + + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="/design"> + Design <span class="caret"></span> + </a> +<ul class="dropdown-menu"> +<li id="nav_design_overview"><a href="/design/architecture.html">Overview</a></li> +<li id="nav_app_needs"><a href="/docs/slider_specs/application_needs.html">Application Needs</a></li> +<li id="nav_spec"><a href="/design/specification/index.html">Expected Behavior</a></li> +<li id="nav_registry"><a href="/design/registry/index.html">Service Registry</a></li> +<li id="nav_ssl_impl"><a href="/design/ssl_implementation.html">SSL Implementation</a></li> +<li id="nav_role_history"><a href="/design/rolehistory.html">Role History</a></li> +</ul> +</li> + + + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="/developing"> + Developing <span class="caret"></span> + </a> +<ul class="dropdown-menu"> +<li id="nav_dev_building"><a href="/developing/building.html">Building</a></li> +<li id="nav_dev_debugging"><a href="/developing/debugging.html">Debugging</a></li> +<li id="nav_dev_testing"><a href="/developing/testing.html">Testing</a></li> +<li id="nav_dev_funtesting"><a href="/developing/functional_tests.html">Functional Testing</a></li> +<li id="nav_dev_manual_testing"><a href="/developing/manual_testing.html">Manual Testing</a></li> +<li id="nav_releasing"><a href="/developing/releasing.html">Releasing</a></li> +</ul> +</li> + + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#"> + Using <span class="caret"></span> + </a> +<ul class="dropdown-menu"> +<li id="nav_use_getting_started"><a href="/docs/getting_started.html">Getting Started</a></li> +<li id="nav_use_manpage"><a href="/docs/manpage.html">Man Page</a></li> +<li role="presentation" class="divider"></li> +<li id="nav_use_app_package"><a href="/docs/slider_specs/index.html">Creating App Packages</a></li> +<li role="presentation" class="divider"></li> +<li id="nav_use_client_config"><a href="/docs/client-configuration.html">Client Configuration</a></li> +<li id="nav_use_exitcodes"><a href="/docs/exitcodes.html">Client Exit Codes</a></li> +<li role="presentation" class="divider"></li> +<li id="nav_use_debugging"><a href="/developing/debugging.html">Debugging</a></li> +<li id="nav_use_troubleshooting"><a href="/docs/troubleshooting.html">Troubleshooting</a></li> +<li role="presentation" class="divider"></li> +<li id="nav_use_high_availability"><a href="/docs/high_availability.html">High Availability</a></li> +<li id="nav_use_security"><a href="/docs/security.html">Security</a></li> +<li id="nav_am_agent_ssl"><a href="/docs/ssl.html">Agent to AM SSL</a></li> +<li id="nav_use_app_configuration"><a href="/docs/configuration/index.html">App Configuration Model</a></li> +<li id="nav_use_examples"><a href="/docs/examples.html">Provider Examples</a></li> +</ul> +</li> + + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#"> + ASF Links <span class="caret"></span> + </a> +<ul class="dropdown-menu"> +<li><a href="http://www.apache.org";><i class="fa fa-external-link"></i> Apache Software Foundation</a></li> +<li><a href="http://www.apache.org/foundation/sponsorship.html";><i class="fa fa-external-link"></i> Sponsorship</a></li> +<li><a href="http://www.apache.org/security/";><i class="fa fa-external-link"></i> Security</a></li> +<li><a href="http://www.apache.org/foundation/thanks.html";><i class="fa fa-external-link"></i> Thanks</a></li> +</ul> +</li> +</ul> + + </div> + +</div> +</nav> + + +<div class="container-fluid"> +<div class="row"> + <div class="col-md-2" id="sidebar"> + <div style="text-align: center"> + <h1><a href="/index.html">Apache Slider (incubating)</a></h1> + <hr> +Latest release: <strong>0.60.0-incubating</strong><br> + <br> + <a id="download-button-sidebar" class="btn btn-success btn-block" href="/downloads/" role="button">Download</a> + </div> + <hr> + <a href="http://incubator.apache.org";><img id="logo" alt="Apache Incubator ™" class="img-responsive" src="http://incubator.apache.org/images/apache-incubator-logo.png"/></a> + </div> + <div class="col-md-8 col-md-offset-1"> + + <div id="content"> + + <h1 class="title"></h1> + + <!--- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<h1 id="apache-slider-specification-of-an-application-instance-revision-1">Apache Slider: Specification of an application instance, revision 1</h1> +<p>This is the original specification of an application instance, including discussion +on a proposed rework.</p> +<ol> +<li><a href="specification.html">Specification</a></li> +<li> +<p><a href="redesign.html">Redesign</a></p> +</li> +<li> +<p><a href="original-hbase.json">Example: current</a></p> +</li> +<li><a href="proposed-hbase.json">Example: proposed</a></li> +</ol> +<p>This design has been supplanted by the <a href="../index.html">version 2.0</a> design.</p> + </div> + + <div id="footer"> + <a alt="Apache Software Foundation" href="http://www.apache.org";> + <img id="asf-logo" alt="Apache Software Foundation" src="/images/feather-small.gif" width="100"> + </a> + <div class="copyright"> + <p> + Copyright © 2011-2014 The Apache Software Foundation, Licensed under + the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>. + Apache Slider, Slider, Apache, the Apache feather logo, and the Apache Incubator + project logo are trademarks of the <a href="http://www.apache.org";>Apache Software Foundation</a>.<br /> + Site created with <a href="http://getbootstrap.com/";>Bootstrap</a> including icons from <a href="http://glyphicons.com/";>GLYPHICONS</a> and <a href="http://fontawesome.io/";>Font Awesome</a>. + </p> + </div> + </div> + </div> + +</div> +</div> + +<script type="text/javascript"> + +</script> +</body> +</html> Added: websites/staging/slider/trunk/content/docs/configuration/revision-1/original-hbase.json ============================================================================== --- websites/staging/slider/trunk/content/docs/configuration/revision-1/original-hbase.json (added) +++ websites/staging/slider/trunk/content/docs/configuration/revision-1/original-hbase.json Wed Mar 25 17:59:38 2015 @@ -0,0 +1,139 @@ +{ + "version": "1.0", + "name": "test_cluster_lifecycle", + "type": "hbase", + "state": 3, + "createTime": 1393512091276, + "updateTime": 1393512117286, + "originConfigurationPath": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/snapshot", + "generatedConfigurationPath": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/generated", + "dataPath": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/database", + "options": { + "zookeeper.port": "2181", + "site.hbase.master.startup.retainassign": "true", + "slider.cluster.application.image.path": "hdfs://sandbox:8020/hbase.tar.gz", + "site.fs.defaultFS": "hdfs://sandbox:8020", + "slider.container.failure.threshold": "5", + "site.fs.default.name": "hdfs://sandbox:8020", + "slider.cluster.directory.permissions": "0770", + "slider.am.monitoring.enabled": "false", + "zookeeper.path": "/yarnapps_slider_stevel_test_cluster_lifecycle", + "slider.tmp.dir": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/tmp/am", + "slider.data.directory.permissions": "0770", + "zookeeper.hosts": "sandbox", + "slider.container.failure.shortlife": "60" + }, + "info": { + "create.hadoop.deployed.info": "(detached from release-2.3.0) @dfe46336fbc6a044bc124392ec06b85", + "create.application.build.info": "Slider Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel", + "create.hadoop.build.info": "2.3.0", + "create.time.millis": "1393512091276", + "create.time": "27 Feb 2014 14:41:31 GMT", + "slider.am.restart.supported": "false", + "live.time": "27 Feb 2014 14:41:56 GMT", + "live.time.millis": "1393512116881", + "status.time": "27 Feb 2014 14:42:08 GMT", + "status.time.millis": "1393512128726", + "yarn.vcores": "32", + "yarn.memory": "2048", + "status.application.build.info": "Slider Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel", + "status.hadoop.build.info": "2.3.0", + "status.hadoop.deployed.info": "bigwheel-m16-2.2.0 @704f1e463ebc4fb89353011407e965" + }, + "statistics": { + "worker": { + "containers.start.started": 0, + "containers.live": 0, + "containers.start.failed": 0, + "containers.active.requests": 0, + "containers.failed": 0, + "containers.completed": 0, + "containers.desired": 0, + "containers.requested": 0 + }, + "slider": { + "containers.unknown.completed": 0, + "containers.start.started": 0, + "containers.live": 1, + "containers.start.failed": 0, + "containers.failed": 0, + "containers.completed": 0, + "containers.surplus": 0 + }, + "master": { + "containers.start.started": 0, + "containers.live": 0, + "containers.start.failed": 0, + "containers.active.requests": 0, + "containers.failed": 0, + "containers.completed": 0, + "containers.desired": 0, + "containers.requested": 0 + } + }, + "status": { + }, + "instances": { + "slider": [ "container_1393511571284_0002_01_000001" ] + }, + "roles": { + "worker": { + "yarn.memory": "768", + "env.MALLOC_ARENA_MAX": "4", + "role.instances": "0", + "role.requested.instances": "0", + "role.name": "worker", + "role.failed.starting.instances": "0", + "role.actual.instances": "0", + "jvm.heapsize": "512M", + "yarn.vcores": "1", + "role.releasing.instances": "0", + "role.failed.instances": "0", + "app.infoport": "0" + }, + "slider": { + "yarn.memory": "256", + "env.MALLOC_ARENA_MAX": "4", + "role.instances": "1", + "role.requested.instances": "0", + "role.name": "slider", + "role.failed.starting.instances": "0", + "role.actual.instances": "1", + "jvm.heapsize": "256M", + "yarn.vcores": "1", + "role.releasing.instances": "0", + "role.failed.instances": "0" + }, + "master": { + "yarn.memory": "1024", + "env.MALLOC_ARENA_MAX": "4", + "role.instances": "0", + "role.requested.instances": "0", + "role.name": "master", + "role.failed.starting.instances": "0", + "role.actual.instances": "0", + "jvm.heapsize": "512M", + "yarn.vcores": "1", + "role.releasing.instances": "0", + "role.failed.instances": "0", + "app.infoport": "0" + } + }, + "clientProperties": { + "fs.defaultFS": "hdfs://sandbox:8020", + "hbase.cluster.distributed": "true", + "hbase.master.info.port": "0", + "hbase.master.port": "0", + "hbase.master.startup.retainassign": "true", + "hbase.regionserver.hlog.tolerable.lowreplication": "1", + "hbase.regionserver.info.port": "0", + "hbase.regionserver.port": "0", + "hbase.rootdir": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/database", + "hbase.tmp.dir": "./hbase-tmp", + "hbase.zookeeper.property.clientPort": "2181", + "hbase.zookeeper.quorum": "sandbox", + "slider.template.origin": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/snapshot/hbase-site.xml", + "slider.unused.option": "1", + "zookeeper.znode.parent": "/yarnapps_slider_stevel_test_cluster_lifecycle" + } +} Added: websites/staging/slider/trunk/content/docs/configuration/revision-1/proposed-hbase.json ============================================================================== --- websites/staging/slider/trunk/content/docs/configuration/revision-1/proposed-hbase.json (added) +++ websites/staging/slider/trunk/content/docs/configuration/revision-1/proposed-hbase.json Wed Mar 25 17:59:38 2015 @@ -0,0 +1,273 @@ +{ + "version": "2.0.0", + "name": "test_cluster_lifecycle", + "valid`": true, + + "slider-internal":{ + "type": "hbase", + "createTime": 1393512091276, + "updateTime": 1393512117286, + "originConfigurationPath": "hdfs://sandbox:8020/user/slider/.slider/cluster/test_cluster_lifecycle/snapshot", + "generatedConfigurationPath": "hdfs://sandbox:8020/user/slider/.slider/cluster/test_cluster_lifecycle/generated", + "dataPath": "hdfs://sandbox:8020/user/slider/.slider/cluster/test_cluster_lifecycle/database", + "slider.tmp.dir": "hdfs://sandbox:8020/user/slider/.slider/cluster/test_cluster_lifecycle/tmp/am", + "slider.cluster.directory.permissions": "0770", + "slider.data.directory.permissions": "0770" + }, + + "options": { + "slider.am.monitoring.enabled": "false", + "slider.cluster.application.image.path": "hdfs://sandbox:8020/hbase.tar.gz", + "slider.container.failure.threshold": "5", + "slider.container.failure.shortlife": "60", + "zookeeper.port": "2181", + "zookeeper.path": "/yarnapps_slider_slider_test_cluster_lifecycle", + "zookeeper.hosts": "sandbox", + "site.hbase.master.startup.retainassign": "true", + "site.fs.defaultFS": "hdfs://sandbox:8020", + "site.fs.default.name": "hdfs://sandbox:8020", + "env.MALLOC_ARENA_MAX": "4", + "site.hbase.master.info.port": "0", + "site.hbase.regionserver.info.port": "0" + }, + + "diagnostics": { + "create.hadoop.deployed.info": "(detached from release-2.3.0) @dfe46336fbc6a044bc124392ec06b85", + "create.application.build.info": "Slider Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by slider", + "create.hadoop.build.info": "2.3.0", + "create.time.millis": "1393512091276", + "create.time": "27 Feb 2014 14:41:31 GMT" + }, + + "info": { + "slider.am.restart.supported": "false", + "live.time": "27 Feb 2014 14:41:56 GMT", + "live.time.millis": "1393512116881", + "status.time": "27 Feb 2014 14:42:08 GMT", + "status.time.millis": "1393512128726", + "yarn.vcores": "32", + "yarn.memory": "2048", + "status.application.build.info": "Slider Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by slider", + "status.hadoop.build.info": "2.3.0", + "status.hadoop.deployed.info": "bigwheel-m16-2.2.0 @704f1e463ebc4fb89353011407e965" + }, + + "statistics": { + + "cluster": { + "containers.unknown.completed": 0, + "containers.start.completed": 3, + "containers.live": 1, + "containers.start.failed": 0, + "containers.failed": 0, + "containers.completed": 0, + "containers.surplus": 0 + + }, + "roles": { + "worker": { + "containers.start.completed": 0, + "containers.live": 2, + "containers.start.failed": 0, + "containers.active.requests": 0, + "containers.failed": 0, + "containers.completed": 0, + "containers.desired": 2, + "containers.requested": 0 + }, + "master": { + "containers.start.completed": 0, + "containers.live": 1, + "containers.start.failed": 0, + "containers.active.requests": 0, + "containers.failed": 0, + "containers.completed": 0, + "containers.desired": 1, + "containers.requested": 0 + } + } + }, + + "instances": { + "slider": [ "container_1393511571284_0002_01_000001" ], + "master": [ "container_1393511571284_0002_01_000003" ], + "worker": [ + "container_1393511571284_0002_01_000002", + "container_1393511571284_0002_01_000004" + ] + }, + + "roles": { + "worker": { + "yarn.memory": "768", + "role.instances": "0", + "role.name": "worker", + "jvm.heapsize": "512M", + "yarn.vcores": "1" + }, + "slider": { + "yarn.memory": "256", + "role.instances": "1", + "role.name": "slider", + "jvm.heapsize": "256M", + "yarn.vcores": "1" + }, + "master": { + "yarn.memory": "1024", + "role.instances": "0", + "role.name": "master", + "jvm.heapsize": "512M", + "yarn.vcores": "1" + } + }, + + + "clientProperties": { + "fs.defaultFS": "hdfs://sandbox:8020", + "hbase.cluster.distributed": "true", + "hbase.master.info.port": "0", + "hbase.master.port": "0", + "hbase.master.startup.retainassign": "true", + "hbase.regionserver.hlog.tolerable.lowreplication": "1", + "hbase.regionserver.info.port": "0", + "hbase.regionserver.port": "0", + "hbase.rootdir": "hdfs://sandbox:8020/user/slider/.slider/cluster/test_cluster_lifecycle/database", + "hbase.tmp.dir": "./hbase-tmp", + "hbase.zookeeper.property.clientPort": "2181", + "hbase.zookeeper.quorum": "sandbox", + "zookeeper.znode.parent": "/yarnapps_slider_slider_test_cluster_lifecycle" + }, + + + "clientfiles": { + "hbase-site.xml": "site information for HBase", + "log4.properties": "log4.property file" + }, + + "provider":{ + "load":0.4, + "urls": { + "master": ["http://node4:28209";], + "worker": ["http://node4:28717";, "http://node6:31268";] + } + }, + + "status": { + "live": { + "worker": { + "container_1394032374441_0001_01_000003": { + "name": "container_1394032374441_0001_01_000003", + "role": "worker", + "roleId": 1, + "createTime": 1394032384451, + "startTime": 1394032384503, + "released": false, + "host": "192.168.1.88", + "state": 3, + "exitCode": 0, + "command": "hbase-0.98.0/bin/hbase --config $PROPAGATED_CONFDIR regionserver start 1><LOG_DIR>/region-server.txt 2>&1 ; ", + "diagnostics": "", + "environment": [ + "HADOOP_USER_NAME=\"slider\"", + "HBASE_LOG_DIR=\"/tmp/slider-slider\"", + "HBASE_HEAPSIZE=\"256\"", + "MALLOC_ARENA_MAX=\"4\"", + "PROPAGATED_CONFDIR=\"$PWD/propagatedconf\"" + ] + }, + "container_1394032374441_0001_01_000002": { + "name": "container_1394032374441_0001_01_000002", + "role": "worker", + "roleId": 1, + "createTime": 1394032384451, + "startTime": 1394032384552, + "released": false, + "host": "192.168.1.86", + "state": 3, + "exitCode": 0, + "command": "hbase-0.98.0/bin/hbase --config $PROPAGATED_CONFDIR regionserver start 1><LOG_DIR>/region-server.txt 2>&1 ; ", + "diagnostics": "", + "environment": [ + "HADOOP_USER_NAME=\"slider\"", + "HBASE_LOG_DIR=\"/tmp/slider-slider\"", + "HBASE_HEAPSIZE=\"256\"", + "MALLOC_ARENA_MAX=\"4\"", + "PROPAGATED_CONFDIR=\"$PWD/propagatedconf\"" + ] + } + }, + "slider": { + "container_1394032374441_0001_01_000001": { + "name": "container_1394032374441_0001_01_000001", + "role": "slider", + "roleId": 0, + "createTime": 0, + "startTime": 0, + "released": false, + "host": "slider-8.local", + "state": 3, + "exitCode": 0, + "command": "", + "diagnostics": "" + } + }, + "master": { + "container_1394032374441_0001_01_000004": { + "name": "container_1394032374441_0001_01_000004", + "role": "master", + "roleId": 2, + "createTime": 1394032384451, + "startTime": 1394032384573, + "released": false, + "host": "192.168.1.86", + "state": 3, + "exitCode": 0, + "command": "hbase-0.98.0/bin/hbase --config $PROPAGATED_CONFDIR master start 1><LOG_DIR>/master.txt 2>&1 ; ", + "diagnostics": "", + "environment": [ + "HADOOP_USER_NAME=\"slider\"", + "HBASE_LOG_DIR=\"/tmp/slider-slider\"", + "HBASE_HEAPSIZE=\"256\"", + "MALLOC_ARENA_MAX=\"4\"", + "PROPAGATED_CONFDIR=\"$PWD/propagatedconf\"" + ] + } + } + }, + "failed": { + + }, + + "rolestatus": { + "worker": { + "role.instances": "2", + "role.requested.instances": "0", + "role.failed.starting.instances": "0", + "role.actual.instances": "2", + "role.releasing.instances": "0", + "role.failed.instances": "1" + }, + "slider": { + "role.instances": "1", + "role.requested.instances": "0", + "role.name": "slider", + "role.actual.instances": "1", + "role.releasing.instances": "0", + "role.failed.instances": "0" + }, + "master": { + "role.instances": "1", + "role.requested.instances": "1", + "role.name": "master", + "role.failed.starting.instances": "0", + "role.actual.instances": "0", + "role.releasing.instances": "0", + "role.failed.instances": "0" + } + } + } + + + + +}
![http://incubator.apache.org/images/apache-incubator-logo.png"/](http://incubator.apache.org/images/apache-incubator-logo.png"/){kind=link}