Added: websites/staging/slider/trunk/content/docs/configuration/revision-1/redesign.html ============================================================================== --- websites/staging/slider/trunk/content/docs/configuration/revision-1/redesign.html (added) +++ websites/staging/slider/trunk/content/docs/configuration/revision-1/redesign.html Wed Mar 25 17:59:38 2015 @@ -0,0 +1,629 @@ +<!DOCTYPE html> +<html lang="en"> +<head> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE- 2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <link href="/css/bootstrap.min.css" rel="stylesheet"> + <link href="/css/bootstrap-theme.min.css" rel="stylesheet"> + <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js";></script> + <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js";></script> + <![endif]--> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js";></script> + <script src="/js/bootstrap.min.js"></script> + <link href="/css/slider.css" rel="stylesheet" type="text/css"> + <link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet"> + <title></title> + +</head> + +<body> +<nav class="navbar navbar-default navbar-fixed-top" role="navigation"> +<div class="container-fluid"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar-items"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="/index.html">Slider</a> + </div> + <div class="collapse navbar-collapse" id="navbar-items"> + <ul class="nav navbar-nav"> + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#"> + Project <span class="caret"></span> + </a> +<ul class="dropdown-menu"> +<li id="nav_index"><a href="/index.html">Home</a></li> +<li id="nav_downloads"><a href="/downloads">Downloads</a></li> +<li id="nav_source"><a href="https://git-wip-us.apache.org/repos/asf/incubator-slider.git";><i class="fa fa-external-link"></i> Source</a></li> +<li><a href="http://www.apache.org/licenses/LICENSE-2.0";><i class="fa fa-external-link"></i> License</a></li> +<li class="divider"></li> +<li class="dropdown-header">Community</li> +<li id="nav_team"><a href="/team.html">Team</a></li> +<li id="nav_mailinglists"><a href="/mailing_lists.html">Mailing Lists</a></li> +<li class="divider"></li> +<li class="dropdown-header">Tools</li> +<li id="nav_jira"><a href="https://issues.apache.org/jira/browse/SLIDER";><i class="fa fa-external-link"></i> Issues</a></li> +<li id="nav_builds"><a href="https://builds.apache.org/job/Slider-develop";><i class="fa fa-external-link"></i> Builds</a></li> +</ul> +</li> + + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="/design"> + Design <span class="caret"></span> + </a> +<ul class="dropdown-menu"> +<li id="nav_design_overview"><a href="/design/architecture.html">Overview</a></li> +<li id="nav_app_needs"><a href="/docs/slider_specs/application_needs.html">Application Needs</a></li> +<li id="nav_spec"><a href="/design/specification/index.html">Expected Behavior</a></li> +<li id="nav_registry"><a href="/design/registry/index.html">Service Registry</a></li> +<li id="nav_ssl_impl"><a href="/design/ssl_implementation.html">SSL Implementation</a></li> +<li id="nav_role_history"><a href="/design/rolehistory.html">Role History</a></li> +</ul> +</li> + + + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="/developing"> + Developing <span class="caret"></span> + </a> +<ul class="dropdown-menu"> +<li id="nav_dev_building"><a href="/developing/building.html">Building</a></li> +<li id="nav_dev_debugging"><a href="/developing/debugging.html">Debugging</a></li> +<li id="nav_dev_testing"><a href="/developing/testing.html">Testing</a></li> +<li id="nav_dev_funtesting"><a href="/developing/functional_tests.html">Functional Testing</a></li> +<li id="nav_dev_manual_testing"><a href="/developing/manual_testing.html">Manual Testing</a></li> +<li id="nav_releasing"><a href="/developing/releasing.html">Releasing</a></li> +</ul> +</li> + + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#"> + Using <span class="caret"></span> + </a> +<ul class="dropdown-menu"> +<li id="nav_use_getting_started"><a href="/docs/getting_started.html">Getting Started</a></li> +<li id="nav_use_manpage"><a href="/docs/manpage.html">Man Page</a></li> +<li role="presentation" class="divider"></li> +<li id="nav_use_app_package"><a href="/docs/slider_specs/index.html">Creating App Packages</a></li> +<li role="presentation" class="divider"></li> +<li id="nav_use_client_config"><a href="/docs/client-configuration.html">Client Configuration</a></li> +<li id="nav_use_exitcodes"><a href="/docs/exitcodes.html">Client Exit Codes</a></li> +<li role="presentation" class="divider"></li> +<li id="nav_use_debugging"><a href="/developing/debugging.html">Debugging</a></li> +<li id="nav_use_troubleshooting"><a href="/docs/troubleshooting.html">Troubleshooting</a></li> +<li role="presentation" class="divider"></li> +<li id="nav_use_high_availability"><a href="/docs/high_availability.html">High Availability</a></li> +<li id="nav_use_security"><a href="/docs/security.html">Security</a></li> +<li id="nav_am_agent_ssl"><a href="/docs/ssl.html">Agent to AM SSL</a></li> +<li id="nav_use_app_configuration"><a href="/docs/configuration/index.html">App Configuration Model</a></li> +<li id="nav_use_examples"><a href="/docs/examples.html">Provider Examples</a></li> +</ul> +</li> + + <li class="dropdown"> + <a class="dropdown-toggle" data-toggle="dropdown" href="#"> + ASF Links <span class="caret"></span> + </a> +<ul class="dropdown-menu"> +<li><a href="http://www.apache.org";><i class="fa fa-external-link"></i> Apache Software Foundation</a></li> +<li><a href="http://www.apache.org/foundation/sponsorship.html";><i class="fa fa-external-link"></i> Sponsorship</a></li> +<li><a href="http://www.apache.org/security/";><i class="fa fa-external-link"></i> Security</a></li> +<li><a href="http://www.apache.org/foundation/thanks.html";><i class="fa fa-external-link"></i> Thanks</a></li> +</ul> +</li> +</ul> + + </div> + +</div> +</nav> + + +<div class="container-fluid"> +<div class="row"> + <div class="col-md-2" id="sidebar"> + <div style="text-align: center"> + <h1><a href="/index.html">Apache Slider (incubating)</a></h1> + <hr> +Latest release: <strong>0.60.0-incubating</strong><br> + <br> + <a id="download-button-sidebar" class="btn btn-success btn-block" href="/downloads/" role="button">Download</a> + </div> + <hr> + <a href="http://incubator.apache.org";><img id="logo" alt="Apache Incubator ™" class="img-responsive" src="http://incubator.apache.org/images/apache-incubator-logo.png"/></a> + </div> + <div class="col-md-8 col-md-offset-1"> + + <div id="content"> + + <h1 class="title"></h1> + + <!--- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<h1 id="apache-slider-cluster-specification">Apache Slider Cluster Specification</h1> +<h3 id="notation">Notation:</h3> +<p>In this document, a full path to a value is represented as a path +<code>options/zookeeper.port</code> ; an assigment as <code>options/zookeeper.port=2181</code>.</p> +<p>A wildcard indicates all entries matching a path: <code>options/zookeeper.*</code> +or <code>/roles/*/yarn.memory</code></p> +<h2 id="history">History</h2> +<p>The Slider cluster specification was implicitly defined in the file +<code>org.apache.slider.api.ClusterDescription</code>. It had a number of roles</p> +<ol> +<li>Persistent representaton of cluster state</li> +<li>Internal model of desired cluster state within the Application Master.</li> +<li>Dynamic representation of current cluster state when the AM +was queried, marshalled over the network as JSON.</li> +<li>Description of updated state when reconfiguring a running cluster.</li> +</ol> +<p>Initially the dynamic status included a complete history of all containers +-this soon highlit some restrictions on the maximum size of a JSON-formatted +string in Hadoop's "classic" RPC: 32K, after which the string was silently +truncated. Accordingly, this history was dropped.</p> +<p>Having moved to Protocol Buffers as the IPC wire format, with a web view +alongside, this history could be reconsidered.</p> +<p>The initial design place most values into the root entry, and relied +on Jaxon introspection to set and retrieve the values -it was a +Java-first specification, with no external specificatin or regression tests.</p> +<p>As the number of entries in the root increased, the design switched to storing +more attributes into specific sections <em>under</em> the root path:</p> +<ul> +<li><code>info</code>: read-only information about the cluster.</li> +<li><code>statistics</code>: Numeric statistics about the cluster</li> +</ul> +<h1 id="sections">Sections</h1> +<h2 id="root">Root</h2> +<p>Contains various string and integer values</p> +<div class="codehilite"><pre>"<span class="n">version</span>"<span class="p">:</span> "1<span class="p">.</span>0"<span class="p">,</span> +"<span class="n">name</span>"<span class="p">:</span> "<span class="n">test_cluster_lifecycle</span>"<span class="p">,</span> +"<span class="n">type</span>"<span class="p">:</span> "<span class="n">hbase</span>"<span class="p">,</span> +"<span class="n">state</span>"<span class="p">:</span> 3<span class="p">,</span> +"<span class="n">createTime</span>"<span class="p">:</span> 1393512091276<span class="p">,</span> +"<span class="n">updateTime</span>"<span class="p">:</span> 1393512117286<span class="p">,</span> +"<span class="n">originConfigurationPath</span>"<span class="p">:</span> "<span class="n">hdfs</span><span class="p">:</span><span class="o">//</span><span class="n">sandbox</span><span class="p">:</span>8020<span class="o">/</span><span class="n">user</span><span class="o">/</span><span class="n">stevel</span><span class="o">/</span><span class="p">.</span><span class="n">slider</span><span class="o">/</span><span class="n">cluster</span><span class="o">/</span><span class="n">test_cluster_lifecycle</span><span class="o">/</span><span class="n">snapshot</span>"<span class="p">,</span> +"<span class="n">generatedConfigurationPath</span>"<span class="p">:</span> "<span class="n">hdfs</span><span class="p">:</span><span class="o">//</span><span class="n">sandbox</span><span class="p">:</span>8020<span class="o">/</span><span class="n">user</span><span class="o">/</span><span class="n">stevel</span><span class="o">/</span><span class="p">.</span><span class="n">slider</span><span class="o">/</span><span class="n">cluster</span><span class="o">/</span><span class="n">test_cluster_lifecycle</span><span class="o">/</span><span class="n">generated</span>"<span class="p">,</span> +"<span class="n">dataPath</span>"<span class="p">:</span> "<span class="n">hdfs</span><span class="p">:</span><span class="o">//</span><span class="n">sandbox</span><span class="p">:</span>8020<span class="o">/</span><span class="n">user</span><span class="o">/</span><span class="n">stevel</span><span class="o">/</span><span class="p">.</span><span class="n">slider</span><span class="o">/</span><span class="n">cluster</span><span class="o">/</span><span class="n">test_cluster_lifecycle</span><span class="o">/</span><span class="n">database</span>"<span class="p">,</span> +</pre></div> + + +<ul> +<li> +<p><code>version</code>: version of the JSON file. Not currently used +to validate version compatibility; at this point in time +releases may not be able to read existing .json files.</p> +</li> +<li> +<p><code>name</code>: cluster name</p> +</li> +<li><code>type</code>: reference to the provider type -this triggers a Hadoop configuration +property lookup to find the implementation classes.</li> +<li> +<p><code>state</code>: an enumeration value of the cluster state.</p> +<div class="codehilite"><pre><span class="n">int</span> <span class="n">STATE_INCOMPLETE</span> <span class="p">=</span> 0<span class="p">;</span> +<span class="n">int</span> <span class="n">STATE_SUBMITTED</span> <span class="p">=</span> 1<span class="p">;</span> +<span class="n">int</span> <span class="n">STATE_CREATED</span> <span class="p">=</span> 2<span class="p">;</span> +<span class="n">int</span> <span class="n">STATE_LIVE</span> <span class="p">=</span> 3<span class="p">;</span> +<span class="n">int</span> <span class="n">STATE_STOPPED</span> <span class="p">=</span> 4<span class="p">;</span> +<span class="n">int</span> <span class="n">STATE_DESTROYED</span> <span class="p">=</span> 5<span class="p">;</span> +</pre></div> + + +</li> +</ul> +<p>Only two states are persisted, "incomplete" and "created", though more + are used internally. + The <code>incomplete</code> state is used during cluster create/build, + allowing an incomplete JSON file to be written + -so minimising the window for race conditions on cluster construction.</p> +<ul> +<li><code>createTime</code> and <code>updateTime</code>: timestamps, informative only. + The <code>createTime</code> value is duplicated in <code>/info/createTimeMillis</code></li> +<li><code>originConfigurationPath</code>, <code>generatedConfigurationPath</code>, <code>dataPath</code> paths +used internally -if changed the cluster may not start.</li> +</ul> +<p><em>Proposed</em>: +1. Move all state bar <code>name</code> and cluster state +into a section <code>/slider-internal</code>. +1. The cluster state is moved from an enum to a simple + boolean, <code>valid</code>, set to true when the cluster JSON + has been fully constructed.</p> +<h2 id="info"><code>/info</code></h2> +<p>Read-only list of information about the application. Generally this is +intended to be used for debugging and testing.</p> +<h3 id="persisted-values-static-information-about-the-file-history">Persisted values: static information about the file history</h3> +<div class="codehilite"><pre>"<span class="n">info</span>" <span class="p">:</span> <span class="p">{</span> + "<span class="n">create</span><span class="p">.</span><span class="n">hadoop</span><span class="p">.</span><span class="n">deployed</span><span class="p">.</span><span class="n">info</span>" <span class="p">:</span> "<span class="p">(</span><span class="n">detached</span> <span class="n">from</span> <span class="n">release</span><span class="o">-</span>2<span class="p">.</span>3<span class="p">.</span>0<span class="p">)</span> <span class="p">@</span><span class="n">dfe46336fbc6a044bc124392ec06b85</span>"<span class="p">,</span> + "<span class="n">create</span><span class="p">.</span><span class="n">application</span><span class="p">.</span><span class="n">build</span><span class="p">.</span><span class="n">info</span>" <span class="p">:</span> "<span class="n">Slider</span> <span class="n">Core</span><span class="o">-</span>0<span class="p">.</span>13<span class="p">.</span>0<span class="o">-</span><span class="n">SNAPSHOT</span> <span class="n">Built</span> <span class="n">against</span> <span class="n">commit</span># 1<span class="n">a94ee4aa1</span> <span class="n">on</span> <span class="n">Java</span> 1<span class="p">.</span>7<span class="p">.</span>0<span class="n">_45</span> <span class="n">by</span> <span class="n">stevel</span>"<span class="p">,</span> + "<span class="n">create</span><span class="p">.</span><span class="n">hadoop</span><span class="p">.</span><span class="n">build</span><span class="p">.</span><span class="n">info</span>" <span class="p">:</span> "2<span class="p">.</span>3<span class="p">.</span>0"<span class="p">,</span> + "<span class="n">create</span><span class="p">.</span><span class="n">time</span><span class="p">.</span><span class="n">millis</span>" <span class="p">:</span> "1393512091276"<span class="p">,</span> +<span class="p">},</span> +</pre></div> + + +<p><em>Proposed</em>: move persisted info K-V pairs to a section <code>/diagnostics</code>.</p> +<h3 id="dynamic-values">Dynamic values:</h3> +<p>whether the AM supports service restart without killing all the containers hosting + the role instances:</p> +<div class="codehilite"><pre>"<span class="n">slider</span><span class="p">.</span><span class="n">am</span><span class="p">.</span><span class="n">restart</span><span class="p">.</span><span class="n">supported</span>" <span class="p">:</span> "<span class="n">false</span>"<span class="p">,</span> +</pre></div> + + +<p>timestamps of the cluster going live, and when the status query was made</p> +<div class="codehilite"><pre>"<span class="n">live</span><span class="p">.</span><span class="n">time</span>" <span class="p">:</span> "27 <span class="n">Feb</span> 2014 14<span class="p">:</span>41<span class="p">:</span>56 <span class="n">GMT</span>"<span class="p">,</span> +"<span class="n">live</span><span class="p">.</span><span class="n">time</span><span class="p">.</span><span class="n">millis</span>" <span class="p">:</span> "1393512116881"<span class="p">,</span> +"<span class="n">status</span><span class="p">.</span><span class="n">time</span>" <span class="p">:</span> "27 <span class="n">Feb</span> 2014 14<span class="p">:</span>42<span class="p">:</span>08 <span class="n">GMT</span>"<span class="p">,</span> +"<span class="n">status</span><span class="p">.</span><span class="n">time</span><span class="p">.</span><span class="n">millis</span>" <span class="p">:</span> "1393512128726"<span class="p">,</span> +</pre></div> + + +<p>yarn data provided to the AM</p> +<div class="codehilite"><pre>"<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>" <span class="p">:</span> "32"<span class="p">,</span> +"<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>" <span class="p">:</span> "2048"<span class="p">,</span> +</pre></div> + + +<p>information about the application and hadoop versions in use. Here + the application was built using Hadoop 2.3.0, but is running against the version + of Hadoop built for HDP-2.</p> +<div class="codehilite"><pre>"<span class="n">status</span><span class="p">.</span><span class="n">application</span><span class="p">.</span><span class="n">build</span><span class="p">.</span><span class="n">info</span>" <span class="p">:</span> "<span class="n">Slider</span> <span class="n">Core</span><span class="o">-</span>0<span class="p">.</span>13<span class="p">.</span>0<span class="o">-</span><span class="n">SNAPSHOT</span> <span class="n">Built</span> <span class="n">against</span> <span class="n">commit</span># 1<span class="n">a94ee4aa1</span> <span class="n">on</span> <span class="n">Java</span> 1<span class="p">.</span>7<span class="p">.</span>0<span class="n">_45</span> <span class="n">by</span> <span class="n">stevel</span>"<span class="p">,</span> +"<span class="n">status</span><span class="p">.</span><span class="n">hadoop</span><span class="p">.</span><span class="n">build</span><span class="p">.</span><span class="n">info</span>" <span class="p">:</span> "2<span class="p">.</span>3<span class="p">.</span>0"<span class="p">,</span> +"<span class="n">status</span><span class="p">.</span><span class="n">hadoop</span><span class="p">.</span><span class="n">deployed</span><span class="p">.</span><span class="n">info</span>" <span class="p">:</span> "<span class="n">bigwheel</span><span class="o">-</span><span class="n">m16</span><span class="o">-</span>2<span class="p">.</span>2<span class="p">.</span>0 <span class="p">@</span>704<span class="n">f1e463ebc4fb89353011407e965</span>" +</pre></div> + + +<p>## <code>instances</code></p> +<p>Information about the live containers in a cluster</p> +<div class="codehilite"><pre> "<span class="n">instances</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">slider</span>"<span class="p">:</span> <span class="p">[</span> "<span class="n">container_1393511571284_0002_01_000001</span>" <span class="p">],</span> + "<span class="n">master</span>"<span class="p">:</span> <span class="p">[</span> "<span class="n">container_1393511571284_0002_01_000003</span>" <span class="p">],</span> + "<span class="n">worker</span>"<span class="p">:</span> <span class="p">[</span> + "<span class="n">container_1393511571284_0002_01_000002</span>"<span class="p">,</span> + "<span class="n">container_1393511571284_0002_01_000004</span>" + <span class="p">]</span> + <span class="p">},</span> +</pre></div> + + +<p>There's no information about location, nor is there any history about containers +that are no longer part of the cluster (i.e. failed & released containers). </p> +<p>It could be possible to include a list of previous containers, +though Slider would need to be selective about how many to store +(or how much detail to retain) on those previous containers.</p> +<p>Perhaps the list could be allowed to grow without limit, but detail +only preserved on the last 100. If more containers fail than that, +there is likely to be a problem which the most recent containers +will also display.</p> +<p><em>Proposed</em> </p> +<ol> +<li>Return to the full serialization of container state -but only for running containers.</li> +<li>Have a list of failed containers, but only include last 8; make it a rolling +buffer. This avoids a significantly failing role to overload the status document.</li> +</ol> +<p>## <code>statistics</code></p> +<p>Statistics on each role. </p> +<p>They can be divided into counters that only increase</p> +<div class="codehilite"><pre>"<span class="n">containers</span><span class="p">.</span><span class="n">start</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 0<span class="p">,</span> +"<span class="n">containers</span><span class="p">.</span><span class="n">start</span><span class="p">.</span><span class="n">failed</span>"<span class="p">:</span> 0<span class="p">,</span> +"<span class="n">containers</span><span class="p">.</span><span class="n">failed</span>"<span class="p">:</span> 0<span class="p">,</span> +"<span class="n">containers</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 0<span class="p">,</span> +"<span class="n">containers</span><span class="p">.</span><span class="n">requested</span>"<span class="p">:</span> 0 +</pre></div> + + +<p>and those that vary depending upon the current state</p> +<div class="codehilite"><pre>"<span class="n">containers</span><span class="p">.</span><span class="n">live</span>"<span class="p">:</span> 0<span class="p">,</span> +"<span class="n">containers</span><span class="p">.</span><span class="n">active</span><span class="p">.</span><span class="n">requests</span>"<span class="p">:</span> 0<span class="p">,</span> +"<span class="n">containers</span><span class="p">.</span><span class="n">desired</span>"<span class="p">:</span> 0<span class="p">,</span> +</pre></div> + + +<ul> +<li> +<p>Propose: move these values out of statistics into some other section, as they +are state, not statistics*</p> +<p>"statistics": { + "worker": { + "containers.start.completed": 0, + "containers.live": 2, + "containers.start.failed": 0, + "containers.active.requests": 0, + "containers.failed": 0, + "containers.completed": 0, + "containers.desired": 2, + "containers.requested": 0 + }, + "slider": { + "containers.unknown.completed": 0, + "containers.start.completed": 3, + "containers.live": 1, + "containers.start.failed": 0, + "containers.failed": 0, + "containers.completed": 0, + "containers.surplus": 0 + }, + "master": { + "containers.start.completed": 0, + "containers.live": 1, + "containers.start.failed": 0, + "containers.active.requests": 0, + "containers.failed": 0, + "containers.completed": 0, + "containers.desired": 1, + "containers.requested": 0 + } + },</p> +</li> +</ul> +<p>The <code>/statistics/slider</code> section is unusual in that it provides the aggregate statistics +of the cluster -this is not obvious. A different name could be used -but +again, there's a risk of clash with or confusion with a role. </p> +<p>Better to have a specific <code>/statistics/cluster</code> element, +and to move the roles' statistics under <code>/statistics/roles</code>:</p> +<div class="codehilite"><pre>"<span class="n">statistics</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">cluster</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">unknown</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 0<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">start</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 3<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">live</span>"<span class="p">:</span> 1<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">start</span><span class="p">.</span><span class="n">failed</span>"<span class="p">:</span> 0<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">failed</span>"<span class="p">:</span> 0<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 0<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">surplus</span>"<span class="p">:</span> 0 + + <span class="p">},</span> + "<span class="n">roles</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">worker</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">start</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 0<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">live</span>"<span class="p">:</span> 2<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">start</span><span class="p">.</span><span class="n">failed</span>"<span class="p">:</span> 0<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">active</span><span class="p">.</span><span class="n">requests</span>"<span class="p">:</span> 0<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">failed</span>"<span class="p">:</span> 0<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 0<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">desired</span>"<span class="p">:</span> 2<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">requested</span>"<span class="p">:</span> 0 + <span class="p">},</span> + "<span class="n">master</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">start</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 0<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">live</span>"<span class="p">:</span> 1<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">start</span><span class="p">.</span><span class="n">failed</span>"<span class="p">:</span> 0<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">active</span><span class="p">.</span><span class="n">requests</span>"<span class="p">:</span> 0<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">failed</span>"<span class="p">:</span> 0<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 0<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">desired</span>"<span class="p">:</span> 1<span class="p">,</span> + "<span class="n">containers</span><span class="p">.</span><span class="n">requested</span>"<span class="p">:</span> 0 + <span class="p">}</span> + <span class="p">}</span> +<span class="p">},</span> +</pre></div> + + +<p>This approach allows extra statistics sections to be added (perhaps +by providers), without any changes to the toplevel section.</p> +<h2 id="options">Options</h2> +<p>A list of options used by Slider and its providers to build up the AM +and the configurations of the deployed service components</p> +<div class="codehilite"><pre>"<span class="n">options</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">zookeeper</span><span class="p">.</span><span class="n">port</span>"<span class="p">:</span> "2181"<span class="p">,</span> + "<span class="n">site</span><span class="p">.</span><span class="n">hbase</span><span class="p">.</span><span class="n">master</span><span class="p">.</span><span class="n">startup</span><span class="p">.</span><span class="n">retainassign</span>"<span class="p">:</span> "<span class="n">true</span>"<span class="p">,</span> + "<span class="n">slider</span><span class="p">.</span><span class="n">cluster</span><span class="p">.</span><span class="n">application</span><span class="p">.</span><span class="n">image</span><span class="p">.</span><span class="n">path</span>"<span class="p">:</span> "<span class="n">hdfs</span><span class="p">:</span><span class="o">//</span><span class="n">sandbox</span><span class="p">:</span>8020<span class="o">/</span><span class="n">hbase</span><span class="p">.</span><span class="n">tar</span><span class="p">.</span><span class="n">gz</span>"<span class="p">,</span> + "<span class="n">site</span><span class="p">.</span><span class="n">fs</span><span class="p">.</span><span class="n">defaultFS</span>"<span class="p">:</span> "<span class="n">hdfs</span><span class="p">:</span><span class="o">//</span><span class="n">sandbox</span><span class="p">:</span>8020"<span class="p">,</span> + "<span class="n">slider</span><span class="p">.</span><span class="n">container</span><span class="p">.</span><span class="n">failure</span><span class="p">.</span><span class="n">threshold</span>"<span class="p">:</span> "5"<span class="p">,</span> + "<span class="n">site</span><span class="p">.</span><span class="n">fs</span><span class="p">.</span><span class="n">default</span><span class="p">.</span><span class="n">name</span>"<span class="p">:</span> "<span class="n">hdfs</span><span class="p">:</span><span class="o">//</span><span class="n">sandbox</span><span class="p">:</span>8020"<span class="p">,</span> + "<span class="n">slider</span><span class="p">.</span><span class="n">cluster</span><span class="p">.</span><span class="n">directory</span><span class="p">.</span><span class="n">permissions</span>"<span class="p">:</span> "0770"<span class="p">,</span> + "<span class="n">slider</span><span class="p">.</span><span class="n">am</span><span class="p">.</span><span class="n">monitoring</span><span class="p">.</span><span class="n">enabled</span>"<span class="p">:</span> "<span class="n">false</span>"<span class="p">,</span> + "<span class="n">zookeeper</span><span class="p">.</span><span class="n">path</span>"<span class="p">:</span> "<span class="o">/</span><span class="n">yarnapps_slider_stevel_test_cluster_lifecycle</span>"<span class="p">,</span> + "<span class="n">slider</span><span class="p">.</span><span class="n">tmp</span><span class="p">.</span><span class="n">dir</span>"<span class="p">:</span> "<span class="n">hdfs</span><span class="p">:</span><span class="o">//</span><span class="n">sandbox</span><span class="p">:</span>8020<span class="o">/</span><span class="n">user</span><span class="o">/</span><span class="n">stevel</span><span class="o">/</span><span class="p">.</span><span class="n">slider</span><span class="o">/</span><span class="n">cluster</span><span class="o">/</span><span class="n">test_cluster_lifecycle</span><span class="o">/</span><span class="n">tmp</span><span class="o">/</span><span class="n">am</span>"<span class="p">,</span> + "<span class="n">slider</span><span class="p">.</span><span class="n">data</span><span class="p">.</span><span class="n">directory</span><span class="p">.</span><span class="n">permissions</span>"<span class="p">:</span> "0770"<span class="p">,</span> + "<span class="n">zookeeper</span><span class="p">.</span><span class="n">hosts</span>"<span class="p">:</span> "<span class="n">sandbox</span>"<span class="p">,</span> + "<span class="n">slider</span><span class="p">.</span><span class="n">container</span><span class="p">.</span><span class="n">failure</span><span class="p">.</span><span class="n">shortlife</span>"<span class="p">:</span> "60" +<span class="p">},</span> +</pre></div> + + +<p>Some for these options have been created by slider itself ("slider.tmp.dir") +for internal use -and are cluster specific. If/when the ability to use +an existing json file as a template for a new cluster is added, having these +options in the configuration will create problems</p> +<h1 id="proposed-changes">Proposed Changes</h1> +<h2 id="move-slider-internal-state-to-slider-internal">Move Slider internal state to <code>/slider-internal</code></h2> +<p>Move all slider "private" data to an internal section,<code>/slider-internal</code> +including those in the toplevel directory and in <code>/options</code></p> +<h2 id="allow-options-and-roles-options-entries-to-take-the-value-null">Allow <code>/options</code> and <code>roles/*/</code> options entries to take the value "null".</h2> +<p>This would be a definition that the value must be defined before the cluster +can start. Provider templates could declare this.</p> +<h2 id="make-client-configuration-retrieval-hierarchical-and-maybe-move-out-of-the">Make client configuration retrieval hierarchical -and maybe move out of the</h2> +<p>status</p> +<p>The current design assumes that it is a -site.xml file being served up. This +does not work for alternate file formats generated by the Provider.</p> +<h2 id="role-options">Role Options</h2> +<p>The <code>/roles/$ROLENAME/</code> clauses each provide options for a +specific role.</p> +<p>This includes +1. <code>role.instances</code>: defines the number of instances of a role to create +1. <code>env.</code> environment variables for launching the container +1. <code>yarn.</code> properties to configure YARN requests. +1. <code>jvm.heapsize</code>: an option supported by some providers to +fix the heap size of a component. +1. <code>app.infoport</code>: an option supported by some providers (e.g. HBase) +to fix the port to which a role (master or worker) binds its web UI.</p> +<div class="codehilite"><pre> "<span class="n">worker</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>"<span class="p">:</span> "768"<span class="p">,</span> + "<span class="n">env</span><span class="p">.</span><span class="n">MALLOC_ARENA_MAX</span>"<span class="p">:</span> "4"<span class="p">,</span> + "<span class="n">role</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "0"<span class="p">,</span> + "<span class="n">role</span><span class="p">.</span><span class="n">name</span>"<span class="p">:</span> "<span class="n">worker</span>"<span class="p">,</span> + "<span class="n">jvm</span><span class="p">.</span><span class="n">heapsize</span>"<span class="p">:</span> "512<span class="n">M</span>"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">app</span><span class="p">.</span><span class="n">infoport</span>"<span class="p">:</span> "0" + <span class="p">},</span> +</pre></div> + + +<p>In a live cluster, the role information also includes status information +about the cluster.</p> +<div class="codehilite"><pre> "<span class="n">master</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>"<span class="p">:</span> "1024"<span class="p">,</span> + "<span class="n">env</span><span class="p">.</span><span class="n">MALLOC_ARENA_MAX</span>"<span class="p">:</span> "4"<span class="p">,</span> + "<span class="n">role</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "0"<span class="p">,</span> + "<span class="n">role</span><span class="p">.</span><span class="n">requested</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "0"<span class="p">,</span> + "<span class="n">role</span><span class="p">.</span><span class="n">name</span>"<span class="p">:</span> "<span class="n">master</span>"<span class="p">,</span> + "<span class="n">role</span><span class="p">.</span><span class="n">failed</span><span class="p">.</span><span class="n">starting</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "0"<span class="p">,</span> + "<span class="n">role</span><span class="p">.</span><span class="n">actual</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "0"<span class="p">,</span> + "<span class="n">jvm</span><span class="p">.</span><span class="n">heapsize</span>"<span class="p">:</span> "512<span class="n">M</span>"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">role</span><span class="p">.</span><span class="n">releasing</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "0"<span class="p">,</span> + "<span class="n">role</span><span class="p">.</span><span class="n">failed</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "0"<span class="p">,</span> + "<span class="n">app</span><span class="p">.</span><span class="n">infoport</span>"<span class="p">:</span> "0" + <span class="p">}</span> +</pre></div> + + +<p>The role <code>slider</code> represents the Slider Application Master itself.</p> +<div class="codehilite"><pre> "<span class="n">slider</span>"<span class="p">:</span> <span class="p">{</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>"<span class="p">:</span> "256"<span class="p">,</span> + "<span class="n">env</span><span class="p">.</span><span class="n">MALLOC_ARENA_MAX</span>"<span class="p">:</span> "4"<span class="p">,</span> + "<span class="n">role</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "1"<span class="p">,</span> + "<span class="n">role</span><span class="p">.</span><span class="n">name</span>"<span class="p">:</span> "<span class="n">slider</span>"<span class="p">,</span> + "<span class="n">jvm</span><span class="p">.</span><span class="n">heapsize</span>"<span class="p">:</span> "256<span class="n">M</span>"<span class="p">,</span> + "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>"<span class="p">:</span> "1"<span class="p">,</span> + <span class="p">},</span> +</pre></div> + + +<h3 id="proposed">Proposed:</h3> +<ol> +<li>move all dynamic role status to its own clauses.</li> +<li>use a simple inheritance model from <code>/options</code></li> +<li>don't allow role entries to alter the cluster state. </li> +</ol> +<h3 id="proposed-clientproperties-continues-return-key-val-pairs">Proposed: <code>/clientProperties</code> continues return Key-val pairs</h3> +<p>The <code>/clientProperties</code> section will remain, with key-val pairs of type +string, the expectation being this is where providers can insert specific +single attributes for client applications.</p> +<p>These values can be converted to application-specific files on the client, +in code -as done today in the Slider CLI-, or via template expansion (beyond +the scope of this document.</p> +<h3 id="proposed-alongside-clientproperties-comes-clientfiles">Proposed: alongside <code>/clientProperties</code> comes <code>/clientfiles</code></h3> +<p>This section will list all files that an application instance can generate +for clients, along with with a description.</p> +<div class="codehilite"><pre>"<span class="o">/</span><span class="n">clientfiles</span><span class="o">/</span><span class="n">hbase</span><span class="o">-</span><span class="n">site</span><span class="p">.</span><span class="n">xml</span>"<span class="p">:</span> "<span class="n">site</span> <span class="n">information</span> <span class="k">for</span> <span class="n">HBase</span>" +"<span class="o">/</span><span class="n">clientfiles</span><span class="o">/</span><span class="n">log4</span><span class="p">.</span><span class="k">properties</span>"<span class="p">:</span> "<span class="n">log4</span><span class="p">.</span><span class="n">property</span> <span class="n">file</span>" +</pre></div> + + +<p>A new CLI command would be added to retrieve a client file. +1. The specific file must be named. +1. If it is not present, an error must be raised. +1. If it is present, it is downloaded and output to the console/to a named +destination file/directory <code>--outfile <file></code> and <code>--outdir <dir></code> +1. If the <code>--list</code> argument is provided, the list of available files is +returned (e.g.) </p> +<div class="codehilite"><pre><span class="n">hbase</span><span class="o">-</span><span class="n">site</span><span class="p">.</span><span class="n">xml</span><span class="p">:</span> <span class="n">site</span> <span class="n">information</span> <span class="k">for</span> <span class="n">HBase</span> +<span class="n">log4</span><span class="p">.</span><span class="k">properties</span><span class="p">:</span> <span class="n">log4</span><span class="p">.</span><span class="n">property</span> <span class="n">file</span> +</pre></div> + + +<p><em>No attempt to parse/process the body of the messages will be returned.</em></p> +<p>In a REST implementation of the client API, /clientconf would be a path +to the list of options; each file a path underneath.</p> +<p>Client configuration file retrieval outside the status completely; +the status just lists the possible values; a separate call returns them.</p> +<p>This will permit binary content to be retrieved, and avoid any marshalling +problems and inefficiencies.</p> +<p>With this change, there will now be two ways to generate client configuration +files</p> +<ul> +<li>Client-side: as today</li> +<li>Server-side: via the provider</li> +</ul> +<p>Client side is more extensible as it allows for arbitrary clients; server-side +is restricted to those files which the application provider is capable of +generating. The advantage of the server-side option is that for those files +about which the provider is aware of, they will be visible through the +REST and Web UIs, so trivially retrieved.</p> +<h3 id="stop-intermixing-role-specification-with-role-current-state">Stop intermixing role specification with role current state</h3> +<p>Create a new section, <code>rolestatus</code>, which lists the current status +of the roles: how many are running vs requested, how many are being +released.</p> +<p>There's some overlap here with the <code>/statistics</code> field, so we should +either merge them or clearly separate the two. Only the <code>role.failed</code> +properties match entries in the statistics -perhaps they should be cut.</p> +<h4 id="provider-specific-status">provider-specific status</h4> +<p>Allow providers to publish information to the status, in their +own section.</p> +<p>There already is support for providers updating the cluster status +in Slider 12.1 and earlier, but it has flaws</p> +<p>A key one is that it is done sychronously on a <code>getStatus()</code> call; +as providers may perform a live query of their status (example, the HBase +provider looks up the Web UI ports published by HBase to zookeeper), +there's overhead, and if the operation blocks (example: when HBase hasn't +ever been deployed and the zookeeper path is empty), then the status +call blocks.</p> +<p><em>Proposed:</em></p> +<ol> +<li>There is a specific <code>/provider</code> section</li> +<li>There's no restriction on what JSON is permitted in this section.</li> +<li>Providers may make their own updates to the application state to read and +write this block -operations that are asynchronous to any status queries.</li> +</ol> + </div> + + <div id="footer"> + <a alt="Apache Software Foundation" href="http://www.apache.org";> + <img id="asf-logo" alt="Apache Software Foundation" src="/images/feather-small.gif" width="100"> + </a> + <div class="copyright"> + <p> + Copyright © 2011-2014 The Apache Software Foundation, Licensed under + the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version 2.0</a>. + Apache Slider, Slider, Apache, the Apache feather logo, and the Apache Incubator + project logo are trademarks of the <a href="http://www.apache.org";>Apache Software Foundation</a>.<br /> + Site created with <a href="http://getbootstrap.com/";>Bootstrap</a> including icons from <a href="http://glyphicons.com/";>GLYPHICONS</a> and <a href="http://fontawesome.io/";>Font Awesome</a>. + </p> + </div> + </div> + </div> + +</div> +</div> + +<script type="text/javascript"> + +</script> +</body> +</html>
![http://incubator.apache.org/images/apache-incubator-logo.png"/](http://incubator.apache.org/images/apache-incubator-logo.png"/){kind=link}
