Added: incubator/nifi/site/trunk/docs/overview.html URL: http://svn.apache.org/viewvc/incubator/nifi/site/trunk/docs/overview.html?rev=1679130&view=auto ============================================================================== --- incubator/nifi/site/trunk/docs/overview.html (added) +++ incubator/nifi/site/trunk/docs/overview.html Wed May 13 03:15:33 2015 @@ -0,0 +1,906 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + <!DOCTYPE html> +<html lang="en"> +<head> +<meta charset="UTF-8"> +<!--[if IE]><meta http-equiv="X-UA-Compatible" content="IE=edge"><![endif]--> +<meta name="viewport" content="width=device-width, initial-scale=1.0"> +<meta name="generator" content="Asciidoctor 1.5.2"> +<meta name="author" content="Apache NiFi Team"> +<title>Apache NiFi Overview</title> +<style> +/* Asciidoctor default stylesheet | MIT License | http://asciidoctor.org */ +/* Copyright (C) 2012-2015 Dan Allen, Ryan Waldron and the Asciidoctor Project + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. */ +/* Remove the comments around the @import statement below when using this as a custom stylesheet */ +@import "https://fonts.googleapis.com/css?family=Open+Sans:300,300italic,400,400italic,600,600italic%7CNoto+Serif:400,400italic,700,700italic%7CDroid+Sans+Mono:400";; +article,aside,details,figcaption,figure,footer,header,hgroup,main,nav,section,summary{display:block} +audio,canvas,video{display:inline-block} +audio:not([controls]){display:none;height:0} +[hidden],template{display:none} +script{display:none!important} +html{font-family:sans-serif;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%} +body{margin:0} +a{background:transparent} +a:focus{outline:thin dotted} +a:active,a:hover{outline:0} +h1{font-size:2em;margin:.67em 0} +abbr[title]{border-bottom:1px dotted} +b,strong{font-weight:bold} +dfn{font-style:italic} +hr{-moz-box-sizing:content-box;box-sizing:content-box;height:0} +mark{background:#ff0;color:#000} +code,kbd,pre,samp{font-family:monospace;font-size:1em} +pre{white-space:pre-wrap} +q{quotes:"\201C" "\201D" "\2018" "\2019"} +small{font-size:80%} +sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline} +sup{top:-.5em} +sub{bottom:-.25em} +img{border:0} +svg:not(:root){overflow:hidden} +figure{margin:0} +fieldset{border:1px solid silver;margin:0 2px;padding:.35em .625em .75em} +legend{border:0;padding:0} +button,input,select,textarea{font-family:inherit;font-size:100%;margin:0} +button,input{line-height:normal} +button,select{text-transform:none} +button,html input[type="button"],input[type="reset"],input[type="submit"]{-webkit-appearance:button;cursor:pointer} +button[disabled],html input[disabled]{cursor:default} +input[type="checkbox"],input[type="radio"]{box-sizing:border-box;padding:0} +input[type="search"]{-webkit-appearance:textfield;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;box-sizing:content-box} +input[type="search"]::-webkit-search-cancel-button,input[type="search"]::-webkit-search-decoration{-webkit-appearance:none} +button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0} +textarea{overflow:auto;vertical-align:top} +table{border-collapse:collapse;border-spacing:0} +*,*:before,*:after{-moz-box-sizing:border-box;-webkit-box-sizing:border-box;box-sizing:border-box} +html,body{font-size:100%} +body{background:#fff;color:rgba(0,0,0,.8);padding:0;margin:0;font-family:"Noto Serif","DejaVu Serif",serif;font-weight:400;font-style:normal;line-height:1;position:relative;cursor:auto} +a:hover{cursor:pointer} +img,object,embed{max-width:100%;height:auto} +object,embed{height:100%} +img{-ms-interpolation-mode:bicubic} +#map_canvas img,#map_canvas embed,#map_canvas object,.map_canvas img,.map_canvas embed,.map_canvas object{max-width:none!important} +.left{float:left!important} +.right{float:right!important} +.text-left{text-align:left!important} +.text-right{text-align:right!important} +.text-center{text-align:center!important} +.text-justify{text-align:justify!important} +.hide{display:none} +.antialiased,body{-webkit-font-smoothing:antialiased} +img{display:inline-block;vertical-align:middle} +textarea{height:auto;min-height:50px} +select{width:100%} +p.lead,.paragraph.lead>p,#preamble>.sectionbody>.paragraph:first-of-type p{font-size:1.21875em;line-height:1.6} +.subheader,.admonitionblock td.content>.title,.audioblock>.title,.exampleblock>.title,.imageblock>.title,.listingblock>.title,.literalblock>.title,.stemblock>.title,.openblock>.title,.paragraph>.title,.quoteblock>.title,table.tableblock>.title,.verseblock>.title,.videoblock>.title,.dlist>.title,.olist>.title,.ulist>.title,.qlist>.title,.hdlist>.title{line-height:1.45;color:#7a2518;font-weight:400;margin-top:0;margin-bottom:.25em} +div,dl,dt,dd,ul,ol,li,h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6,pre,form,p,blockquote,th,td{margin:0;padding:0;direction:ltr} +a{color:#2156a5;text-decoration:underline;line-height:inherit} +a:hover,a:focus{color:#1d4b8f} +a img{border:none} +p{font-family:inherit;font-weight:400;font-size:1em;line-height:1.6;margin-bottom:1.25em;text-rendering:optimizeLegibility} +p aside{font-size:.875em;line-height:1.35;font-style:italic} +h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{font-family:"Open Sans","DejaVu Sans",sans-serif;font-weight:300;font-style:normal;color:#ba3925;text-rendering:optimizeLegibility;margin-top:1em;margin-bottom:.5em;line-height:1.0125em} +h1 small,h2 small,h3 small,#toctitle small,.sidebarblock>.content>.title small,h4 small,h5 small,h6 small{font-size:60%;color:#e99b8f;line-height:0} +h1{font-size:2.125em} +h2{font-size:1.6875em} +h3,#toctitle,.sidebarblock>.content>.title{font-size:1.375em} +h4,h5{font-size:1.125em} +h6{font-size:1em} +hr{border:solid #ddddd8;border-width:1px 0 0;clear:both;margin:1.25em 0 1.1875em;height:0} +em,i{font-style:italic;line-height:inherit} +strong,b{font-weight:bold;line-height:inherit} +small{font-size:60%;line-height:inherit} +code{font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;font-weight:400;color:rgba(0,0,0,.9);padding-right: 1px;} +ul,ol,dl{font-size:1em;line-height:1.6;margin-bottom:1.25em;list-style-position:outside;font-family:inherit} +ul,ol,ul.no-bullet,ol.no-bullet{margin-left:1.5em} +ul li ul,ul li ol{margin-left:1.25em;margin-bottom:0;font-size:1em} +ul.square li ul,ul.circle li ul,ul.disc li ul{list-style:inherit} +ul.square{list-style-type:square} +ul.circle{list-style-type:circle} +ul.disc{list-style-type:disc} +ul.no-bullet{list-style:none} +ol li ul,ol li ol{margin-left:1.25em;margin-bottom:0} +dl dt{margin-bottom:.3125em;font-weight:bold} +dl dd{margin-bottom:1.25em} +abbr,acronym{text-transform:uppercase;font-size:90%;color:rgba(0,0,0,.8);border-bottom:1px dotted #ddd;cursor:help} +abbr{text-transform:none} +blockquote{margin:0 0 1.25em;padding:.5625em 1.25em 0 1.1875em;border-left:1px solid #ddd} +blockquote cite{display:block;font-size:.9375em;color:rgba(0,0,0,.6)} +blockquote cite:before{content:"\2014 \0020"} +blockquote cite a,blockquote cite a:visited{color:rgba(0,0,0,.6)} +blockquote,blockquote p{line-height:1.6;color:rgba(0,0,0,.85)} +@media only screen and (min-width:768px){h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{line-height:1.2} +h1{font-size:2.75em} +h2{font-size:2.3125em} +h3,#toctitle,.sidebarblock>.content>.title{font-size:1.6875em} +h4{font-size:1.4375em}}table{background:#fff;margin-bottom:1.25em;border:solid 1px #dedede} +table thead,table tfoot{background:#f7f8f7;font-weight:bold} +table thead tr th,table thead tr td,table tfoot tr th,table tfoot tr td{padding:.5em .625em .625em;font-size:inherit;color:rgba(0,0,0,.8);text-align:left} +table tr th,table tr td{padding:.5625em .625em;font-size:inherit;color:rgba(0,0,0,.8)} +table tr.even,table tr.alt,table tr:nth-of-type(even){background:#f8f8f7} +table thead tr th,table tfoot tr th,table tbody tr td,table tr td,table tfoot tr td{display:table-cell;line-height:1.6} +h1,h2,h3,#toctitle,.sidebarblock>.content>.title,h4,h5,h6{line-height:1.2;word-spacing:-.05em} +h1 strong,h2 strong,h3 strong,#toctitle strong,.sidebarblock>.content>.title strong,h4 strong,h5 strong,h6 strong{font-weight:400} +.clearfix:before,.clearfix:after,.float-group:before,.float-group:after{content:" ";display:table} +.clearfix:after,.float-group:after{clear:both} +*:not(pre)>code{font-size:.9375em;font-style:normal!important;letter-spacing:0;word-spacing:-.15em;background-color:#f7f7f8;-webkit-border-radius:4px;border-radius:4px;line-height:1.45;text-rendering:optimizeSpeed} +pre,pre>code{line-height:1.45;color:rgba(0,0,0,.9);font-family:"Droid Sans Mono","DejaVu Sans Mono",monospace;font-weight:400;text-rendering:optimizeSpeed} +.keyseq{color:rgba(51,51,51,.8)} +kbd{display:inline-block;color:rgba(0,0,0,.8);font-size:.75em;line-height:1.4;background-color:#f7f7f7;border:1px solid #ccc;-webkit-border-radius:3px;border-radius:3px;-webkit-box-shadow:0 1px 0 rgba(0,0,0,.2),0 0 0 .1em white inset;box-shadow:0 1px 0 rgba(0,0,0,.2),0 0 0 .1em #fff inset;margin:-.15em .15em 0 .15em;padding:.2em .6em .2em .5em;vertical-align:middle;white-space:nowrap} +.keyseq kbd:first-child{margin-left:0} +.keyseq kbd:last-child{margin-right:0} +.menuseq,.menu{color:rgba(0,0,0,.8)} +b.button:before,b.button:after{position:relative;top:-1px;font-weight:400} +b.button:before{content:"[";padding:0 3px 0 2px} +b.button:after{content:"]";padding:0 2px 0 3px} +p a>code:hover{color:rgba(0,0,0,.9)} +#header,#content,#footnotes,#footer{width:100%;margin-left:auto;margin-right:auto;margin-top:0;margin-bottom:0;max-width:62.5em;*zoom:1;position:relative;padding-left:.9375em;padding-right:.9375em} +#header:before,#header:after,#content:before,#content:after,#footnotes:before,#footnotes:after,#footer:before,#footer:after{content:" ";display:table} +#header:after,#content:after,#footnotes:after,#footer:after{clear:both} +#content{margin-top:1.25em} +#content:before{content:none} +#header>h1:first-child{color:rgba(0,0,0,.85);margin-top:2.25rem;margin-bottom:0} +#header>h1:first-child+#toc{margin-top:8px;border-top:1px solid #ddddd8} +#header>h1:only-child,body.toc2 #header>h1:nth-last-child(2){border-bottom:1px solid #ddddd8;padding-bottom:8px} +#header .details{border-bottom:1px solid #ddddd8;line-height:1.45;padding-top:.25em;padding-bottom:.25em;padding-left:.25em;color:rgba(0,0,0,.6);display:-ms-flexbox;display:-webkit-flex;display:flex;-ms-flex-flow:row wrap;-webkit-flex-flow:row wrap;flex-flow:row wrap} +#header .details span:first-child{margin-left:-.125em} +#header .details span.email a{color:rgba(0,0,0,.85)} +#header .details br{display:none} +#header .details br+span:before{content:"\00a0\2013\00a0"} +#header .details br+span.author:before{content:"\00a0\22c5\00a0";color:rgba(0,0,0,.85)} +#header .details br+span#revremark:before{content:"\00a0|\00a0"} +#header #revnumber{text-transform:capitalize} +#header #revnumber:after{content:"\00a0"} +#content>h1:first-child:not([class]){color:rgba(0,0,0,.85);border-bottom:1px solid #ddddd8;padding-bottom:8px;margin-top:0;padding-top:1rem;margin-bottom:1.25rem} +#toc{border-bottom:1px solid #efefed;padding-bottom:.5em} +#toc>ul{margin-left:.125em} +#toc ul.sectlevel0>li>a{font-style:italic} +#toc ul.sectlevel0 ul.sectlevel1{margin:.5em 0} +#toc ul{font-family:"Open Sans","DejaVu Sans",sans-serif;list-style-type:none} +#toc a{text-decoration:none} +#toc a:active{text-decoration:underline} +#toctitle{color:#7a2518;font-size:1.2em} +@media only screen and (min-width:768px){#toctitle{font-size:1.375em} +body.toc2{padding-left:15em;padding-right:0} +#toc.toc2{margin-top:0!important;background-color:#f8f8f7;position:fixed;width:15em;left:0;top:0;border-right:1px solid #efefed;border-top-width:0!important;border-bottom-width:0!important;z-index:1000;padding:1.25em 1em;height:100%;overflow:auto} +#toc.toc2 #toctitle{margin-top:0;font-size:1.2em} +#toc.toc2>ul{font-size:.9em;margin-bottom:0} +#toc.toc2 ul ul{margin-left:0;padding-left:1em} +#toc.toc2 ul.sectlevel0 ul.sectlevel1{padding-left:0;margin-top:.5em;margin-bottom:.5em} +body.toc2.toc-right{padding-left:0;padding-right:15em} +body.toc2.toc-right #toc.toc2{border-right-width:0;border-left:1px solid #efefed;left:auto;right:0}}@media only screen and (min-width:1280px){body.toc2{padding-left:20em;padding-right:0} +#toc.toc2{width:20em} +#toc.toc2 #toctitle{font-size:1.375em} +#toc.toc2>ul{font-size:.95em} +#toc.toc2 ul ul{padding-left:1.25em} +body.toc2.toc-right{padding-left:0;padding-right:20em}}#content #toc{border-style:solid;border-width:1px;border-color:#e0e0dc;margin-bottom:1.25em;padding:1.25em;background:#f8f8f7;-webkit-border-radius:4px;border-radius:4px} +#content #toc>:first-child{margin-top:0} +#content #toc>:last-child{margin-bottom:0} +#footer{max-width:100%;background-color:rgba(0,0,0,.8);padding:1.25em} +#footer-text{color:rgba(255,255,255,.8);line-height:1.44} +.sect1{padding-bottom:.625em} +@media only screen and (min-width:768px){.sect1{padding-bottom:1.25em}}.sect1+.sect1{border-top:1px solid #efefed} +#content h1>a.anchor,h2>a.anchor,h3>a.anchor,#toctitle>a.anchor,.sidebarblock>.content>.title>a.anchor,h4>a.anchor,h5>a.anchor,h6>a.anchor{position:absolute;z-index:1001;width:1.5ex;margin-left:-1.5ex;display:block;text-decoration:none!important;visibility:hidden;text-align:center;font-weight:400} +#content h1>a.anchor:before,h2>a.anchor:before,h3>a.anchor:before,#toctitle>a.anchor:before,.sidebarblock>.content>.title>a.anchor:before,h4>a.anchor:before,h5>a.anchor:before,h6>a.anchor:before{content:"\00A7";font-size:.85em;display:block;padding-top:.1em} +#content h1:hover>a.anchor,#content h1>a.anchor:hover,h2:hover>a.anchor,h2>a.anchor:hover,h3:hover>a.anchor,#toctitle:hover>a.anchor,.sidebarblock>.content>.title:hover>a.anchor,h3>a.anchor:hover,#toctitle>a.anchor:hover,.sidebarblock>.content>.title>a.anchor:hover,h4:hover>a.anchor,h4>a.anchor:hover,h5:hover>a.anchor,h5>a.anchor:hover,h6:hover>a.anchor,h6>a.anchor:hover{visibility:visible} +#content h1>a.link,h2>a.link,h3>a.link,#toctitle>a.link,.sidebarblock>.content>.title>a.link,h4>a.link,h5>a.link,h6>a.link{color:#ba3925;text-decoration:none} +#content h1>a.link:hover,h2>a.link:hover,h3>a.link:hover,#toctitle>a.link:hover,.sidebarblock>.content>.title>a.link:hover,h4>a.link:hover,h5>a.link:hover,h6>a.link:hover{color:#a53221} +.audioblock,.imageblock,.literalblock,.listingblock,.stemblock,.videoblock{margin-bottom:1.25em} +.admonitionblock td.content>.title,.audioblock>.title,.exampleblock>.title,.imageblock>.title,.listingblock>.title,.literalblock>.title,.stemblock>.title,.openblock>.title,.paragraph>.title,.quoteblock>.title,table.tableblock>.title,.verseblock>.title,.videoblock>.title,.dlist>.title,.olist>.title,.ulist>.title,.qlist>.title,.hdlist>.title{text-rendering:optimizeLegibility;text-align:left;font-family:"Noto Serif","DejaVu Serif",serif;font-size:1rem;font-style:italic} +table.tableblock>caption.title{white-space:nowrap;overflow:visible;max-width:0} +.paragraph.lead>p,#preamble>.sectionbody>.paragraph:first-of-type p{color:rgba(0,0,0,.85)} +table.tableblock #preamble>.sectionbody>.paragraph:first-of-type p{font-size:inherit} +.admonitionblock>table{border-collapse:separate;border:0;background:none;width:100%} +.admonitionblock>table td.icon{text-align:center;width:80px} +.admonitionblock>table td.icon img{max-width:none} +.admonitionblock>table td.icon .title{font-weight:bold;font-family:"Open Sans","DejaVu Sans",sans-serif;text-transform:uppercase} +.admonitionblock>table td.content{padding-left:1.125em;padding-right:1.25em;border-left:1px solid #ddddd8;color:rgba(0,0,0,.6)} +.admonitionblock>table td.content>:last-child>:last-child{margin-bottom:0} +.exampleblock>.content{border-style:solid;border-width:1px;border-color:#e6e6e6;margin-bottom:1.25em;padding:1.25em;background:#fff;-webkit-border-radius:4px;border-radius:4px} +.exampleblock>.content>:first-child{margin-top:0} +.exampleblock>.content>:last-child{margin-bottom:0} +.sidebarblock{border-style:solid;border-width:1px;border-color:#e0e0dc;margin-bottom:1.25em;padding:1.25em;background:#f8f8f7;-webkit-border-radius:4px;border-radius:4px} +.sidebarblock>:first-child{margin-top:0} +.sidebarblock>:last-child{margin-bottom:0} +.sidebarblock>.content>.title{color:#7a2518;margin-top:0;text-align:center} +.exampleblock>.content>:last-child>:last-child,.exampleblock>.content .olist>ol>li:last-child>:last-child,.exampleblock>.content .ulist>ul>li:last-child>:last-child,.exampleblock>.content .qlist>ol>li:last-child>:last-child,.sidebarblock>.content>:last-child>:last-child,.sidebarblock>.content .olist>ol>li:last-child>:last-child,.sidebarblock>.content .ulist>ul>li:last-child>:last-child,.sidebarblock>.content .qlist>ol>li:last-child>:last-child{margin-bottom:0} +.literalblock pre,.listingblock pre:not(.highlight),.listingblock pre[class="highlight"],.listingblock pre[class^="highlight "],.listingblock pre.CodeRay,.listingblock pre.prettyprint{background:#f7f7f8} +.sidebarblock .literalblock pre,.sidebarblock .listingblock pre:not(.highlight),.sidebarblock .listingblock pre[class="highlight"],.sidebarblock .listingblock pre[class^="highlight "],.sidebarblock .listingblock pre.CodeRay,.sidebarblock .listingblock pre.prettyprint{background:#f2f1f1} +.literalblock pre,.literalblock pre[class],.listingblock pre,.listingblock pre[class]{-webkit-border-radius:4px;border-radius:4px;word-wrap:break-word;padding:1em;font-size:.8125em} +.literalblock pre.nowrap,.literalblock pre[class].nowrap,.listingblock pre.nowrap,.listingblock pre[class].nowrap{overflow-x:auto;white-space:pre;word-wrap:normal} +@media only screen and (min-width:768px){.literalblock pre,.literalblock pre[class],.listingblock pre,.listingblock pre[class]{font-size:.90625em}}@media only screen and (min-width:1280px){.literalblock pre,.literalblock pre[class],.listingblock pre,.listingblock pre[class]{font-size:1em}}.literalblock.output pre{color:#f7f7f8;background-color:rgba(0,0,0,.9)} +.listingblock pre.highlightjs{padding:0} +.listingblock pre.highlightjs>code{padding:1em;-webkit-border-radius:4px;border-radius:4px} +.listingblock pre.prettyprint{border-width:0} +.listingblock>.content{position:relative} +.listingblock code[data-lang]:before{display:none;content:attr(data-lang);position:absolute;font-size:.75em;top:.425rem;right:.5rem;line-height:1;text-transform:uppercase;color:#999} +.listingblock:hover code[data-lang]:before{display:block} +.listingblock.terminal pre .command:before{content:attr(data-prompt);padding-right:.5em;color:#999} +.listingblock.terminal pre .command:not([data-prompt]):before{content:"$"} +table.pyhltable{border-collapse:separate;border:0;margin-bottom:0;background:none} +table.pyhltable td{vertical-align:top;padding-top:0;padding-bottom:0} +table.pyhltable td.code{padding-left:.75em;padding-right:0} +pre.pygments .lineno,table.pyhltable td:not(.code){color:#999;padding-left:0;padding-right:.5em;border-right:1px solid #ddddd8} +pre.pygments .lineno{display:inline-block;margin-right:.25em} +table.pyhltable .linenodiv{background:none!important;padding-right:0!important} +.quoteblock{margin:0 1em 1.25em 1.5em;display:table} +.quoteblock>.title{margin-left:-1.5em;margin-bottom:.75em} +.quoteblock blockquote,.quoteblock blockquote p{color:rgba(0,0,0,.85);font-size:1.15rem;line-height:1.75;word-spacing:.1em;letter-spacing:0;font-style:italic;text-align:justify} +.quoteblock blockquote{margin:0;padding:0;border:0} +.quoteblock blockquote:before{content:"\201c";float:left;font-size:2.75em;font-weight:bold;line-height:.6em;margin-left:-.6em;color:#7a2518;text-shadow:0 1px 2px rgba(0,0,0,.1)} +.quoteblock blockquote>.paragraph:last-child p{margin-bottom:0} +.quoteblock .attribution{margin-top:.5em;margin-right:.5ex;text-align:right} +.quoteblock .quoteblock{margin-left:0;margin-right:0;padding:.5em 0;border-left:3px solid rgba(0,0,0,.6)} +.quoteblock .quoteblock blockquote{padding:0 0 0 .75em} +.quoteblock .quoteblock blockquote:before{display:none} +.verseblock{margin:0 1em 1.25em 1em} +.verseblock pre{font-family:"Open Sans","DejaVu Sans",sans;font-size:1.15rem;color:rgba(0,0,0,.85);font-weight:300;text-rendering:optimizeLegibility} +.verseblock pre strong{font-weight:400} +.verseblock .attribution{margin-top:1.25rem;margin-left:.5ex} +.quoteblock .attribution,.verseblock .attribution{font-size:.9375em;line-height:1.45;font-style:italic} +.quoteblock .attribution br,.verseblock .attribution br{display:none} +.quoteblock .attribution cite,.verseblock .attribution cite{display:block;letter-spacing:-.05em;color:rgba(0,0,0,.6)} +.quoteblock.abstract{margin:0 0 1.25em 0;display:block} +.quoteblock.abstract blockquote,.quoteblock.abstract blockquote p{text-align:left;word-spacing:0} +.quoteblock.abstract blockquote:before,.quoteblock.abstract blockquote p:first-of-type:before{display:none} +table.tableblock{max-width:100%;border-collapse:separate} +table.tableblock td>.paragraph:last-child p>p:last-child,table.tableblock th>p:last-child,table.tableblock td>p:last-child{margin-bottom:0} +table.spread{width:100%} +table.tableblock,th.tableblock,td.tableblock{border:0 solid #dedede} +table.grid-all th.tableblock,table.grid-all td.tableblock{border-width:0 1px 1px 0} +table.grid-all tfoot>tr>th.tableblock,table.grid-all tfoot>tr>td.tableblock{border-width:1px 1px 0 0} +table.grid-cols th.tableblock,table.grid-cols td.tableblock{border-width:0 1px 0 0} +table.grid-all *>tr>.tableblock:last-child,table.grid-cols *>tr>.tableblock:last-child{border-right-width:0} +table.grid-rows th.tableblock,table.grid-rows td.tableblock{border-width:0 0 1px 0} +table.grid-all tbody>tr:last-child>th.tableblock,table.grid-all tbody>tr:last-child>td.tableblock,table.grid-all thead:last-child>tr>th.tableblock,table.grid-rows tbody>tr:last-child>th.tableblock,table.grid-rows tbody>tr:last-child>td.tableblock,table.grid-rows thead:last-child>tr>th.tableblock{border-bottom-width:0} +table.grid-rows tfoot>tr>th.tableblock,table.grid-rows tfoot>tr>td.tableblock{border-width:1px 0 0 0} +table.frame-all{border-width:1px} +table.frame-sides{border-width:0 1px} +table.frame-topbot{border-width:1px 0} +th.halign-left,td.halign-left{text-align:left} +th.halign-right,td.halign-right{text-align:right} +th.halign-center,td.halign-center{text-align:center} +th.valign-top,td.valign-top{vertical-align:top} +th.valign-bottom,td.valign-bottom{vertical-align:bottom} +th.valign-middle,td.valign-middle{vertical-align:middle} +table thead th,table tfoot th{font-weight:bold} +tbody tr th{display:table-cell;line-height:1.6;background:#f7f8f7} +tbody tr th,tbody tr th p,tfoot tr th,tfoot tr th p{color:rgba(0,0,0,.8);font-weight:bold} +p.tableblock>code:only-child{background:none;padding:0} +p.tableblock{font-size:1em} +td>div.verse{white-space:pre} +ol{margin-left:1.75em} +ul li ol{margin-left:1.5em} +dl dd{margin-left:1.125em} +dl dd:last-child,dl dd:last-child>:last-child{margin-bottom:0} +ol>li p,ul>li p,ul dd,ol dd,.olist .olist,.ulist .ulist,.ulist .olist,.olist .ulist{margin-bottom:.625em} +ul.unstyled,ol.unnumbered,ul.checklist,ul.none{list-style-type:none} +ul.unstyled,ol.unnumbered,ul.checklist{margin-left:.625em} +ul.checklist li>p:first-child>.fa-square-o:first-child,ul.checklist li>p:first-child>.fa-check-square-o:first-child{width:1em;font-size:.85em} +ul.checklist li>p:first-child>input[type="checkbox"]:first-child{width:1em;position:relative;top:1px} +ul.inline{margin:0 auto .625em auto;margin-left:-1.375em;margin-right:0;padding:0;list-style:none;overflow:hidden} +ul.inline>li{list-style:none;float:left;margin-left:1.375em;display:block} +ul.inline>li>*{display:block} +.unstyled dl dt{font-weight:400;font-style:normal} +ol.arabic{list-style-type:decimal} +ol.decimal{list-style-type:decimal-leading-zero} +ol.loweralpha{list-style-type:lower-alpha} +ol.upperalpha{list-style-type:upper-alpha} +ol.lowerroman{list-style-type:lower-roman} +ol.upperroman{list-style-type:upper-roman} +ol.lowergreek{list-style-type:lower-greek} +.hdlist>table,.colist>table{border:0;background:none} +.hdlist>table>tbody>tr,.colist>table>tbody>tr{background:none} +td.hdlist1{padding-right:.75em;font-weight:bold} +td.hdlist1,td.hdlist2{vertical-align:top} +.literalblock+.colist,.listingblock+.colist{margin-top:-.5em} +.colist>table tr>td:first-of-type{padding:0 .75em;line-height:1} +.colist>table tr>td:last-of-type{padding:.25em 0} +.thumb,.th{line-height:0;display:inline-block;border:solid 4px #fff;-webkit-box-shadow:0 0 0 1px #ddd;box-shadow:0 0 0 1px #ddd} +.imageblock.left,.imageblock[style*="float: left"]{margin:.25em .625em 1.25em 0} +.imageblock.right,.imageblock[style*="float: right"]{margin:.25em 0 1.25em .625em} +.imageblock>.title{margin-bottom:0} +.imageblock.thumb,.imageblock.th{border-width:6px} +.imageblock.thumb>.title,.imageblock.th>.title{padding:0 .125em} +.image.left,.image.right{margin-top:.25em;margin-bottom:.25em;display:inline-block;line-height:0} +.image.left{margin-right:.625em} +.image.right{margin-left:.625em} +a.image{text-decoration:none} +span.footnote,span.footnoteref{vertical-align:super;font-size:.875em} +span.footnote a,span.footnoteref a{text-decoration:none} +span.footnote a:active,span.footnoteref a:active{text-decoration:underline} +#footnotes{padding-top:.75em;padding-bottom:.75em;margin-bottom:.625em} +#footnotes hr{width:20%;min-width:6.25em;margin:-.25em 0 .75em 0;border-width:1px 0 0 0} +#footnotes .footnote{padding:0 .375em;line-height:1.3;font-size:.875em;margin-left:1.2em;text-indent:-1.2em;margin-bottom:.2em} +#footnotes .footnote a:first-of-type{font-weight:bold;text-decoration:none} +#footnotes .footnote:last-of-type{margin-bottom:0} +#content #footnotes{margin-top:-.625em;margin-bottom:0;padding:.75em 0} +.gist .file-data>table{border:0;background:#fff;width:100%;margin-bottom:0} +.gist .file-data>table td.line-data{width:99%} +div.unbreakable{page-break-inside:avoid} +.big{font-size:larger} +.small{font-size:smaller} +.underline{text-decoration:underline} +.overline{text-decoration:overline} +.line-through{text-decoration:line-through} +.aqua{color:#00bfbf} +.aqua-background{background-color:#00fafa} +.black{color:#000} +.black-background{background-color:#000} +.blue{color:#0000bf} +.blue-background{background-color:#0000fa} +.fuchsia{color:#bf00bf} +.fuchsia-background{background-color:#fa00fa} +.gray{color:#606060} +.gray-background{background-color:#7d7d7d} +.green{color:#006000} +.green-background{background-color:#007d00} +.lime{color:#00bf00} +.lime-background{background-color:#00fa00} +.maroon{color:#600000} +.maroon-background{background-color:#7d0000} +.navy{color:#000060} +.navy-background{background-color:#00007d} +.olive{color:#606000} +.olive-background{background-color:#7d7d00} +.purple{color:#600060} +.purple-background{background-color:#7d007d} +.red{color:#bf0000} +.red-background{background-color:#fa0000} +.silver{color:#909090} +.silver-background{background-color:#bcbcbc} +.teal{color:#006060} +.teal-background{background-color:#007d7d} +.white{color:#bfbfbf} +.white-background{background-color:#fafafa} +.yellow{color:#bfbf00} +.yellow-background{background-color:#fafa00} +span.icon>.fa{cursor:default} +.admonitionblock td.icon [class^="fa icon-"]{font-size:2.5em;text-shadow:1px 1px 2px rgba(0,0,0,.5);cursor:default} +.admonitionblock td.icon .icon-note:before{content:"\f05a";color:#19407c} +.admonitionblock td.icon .icon-tip:before{content:"\f0eb";text-shadow:1px 1px 2px rgba(155,155,0,.8);color:#111} +.admonitionblock td.icon .icon-warning:before{content:"\f071";color:#bf6900} +.admonitionblock td.icon .icon-caution:before{content:"\f06d";color:#bf3400} +.admonitionblock td.icon .icon-important:before{content:"\f06a";color:#bf0000} +.conum[data-value]{display:inline-block;color:#fff!important;background-color:rgba(0,0,0,.8);-webkit-border-radius:100px;border-radius:100px;text-align:center;font-size:.75em;width:1.67em;height:1.67em;line-height:1.67em;font-family:"Open Sans","DejaVu Sans",sans-serif;font-style:normal;font-weight:bold} +.conum[data-value] *{color:#fff!important} +.conum[data-value]+b{display:none} +.conum[data-value]:after{content:attr(data-value)} +pre .conum[data-value]{position:relative;top:-.125em} +b.conum *{color:inherit!important} +.conum:not([data-value]):empty{display:none} +h1,h2{letter-spacing:-.01em} +dt,th.tableblock,td.content{text-rendering:optimizeLegibility} +p,td.content{letter-spacing:-.01em} +p strong,td.content strong{letter-spacing:-.005em} +p,blockquote,dt,td.content{font-size:1.0625rem} +p{margin-bottom:1.25rem} +.sidebarblock p,.sidebarblock dt,.sidebarblock td.content,p.tableblock{font-size:1em} +.exampleblock>.content{background-color:#fffef7;border-color:#e0e0dc;-webkit-box-shadow:0 1px 4px #e0e0dc;box-shadow:0 1px 4px #e0e0dc} +.print-only{display:none!important} +@media print{@page{margin:1.25cm .75cm} +*{-webkit-box-shadow:none!important;box-shadow:none!important;text-shadow:none!important} +a{color:inherit!important;text-decoration:underline!important} +a.bare,a[href^="#"],a[href^="mailto:"]{text-decoration:none!important} +a[href^="http:"]:not(.bare):after,a[href^="https:"]:not(.bare):after{content:"(" attr(href) ")";display:inline-block;font-size:.875em;padding-left:.25em} +abbr[title]:after{content:" (" attr(title) ")"} +pre,blockquote,tr,img{page-break-inside:avoid} +thead{display:table-header-group} +img{max-width:100%!important} +p,blockquote,dt,td.content{font-size:1em;orphans:3;widows:3} +h2,h3,#toctitle,.sidebarblock>.content>.title{page-break-after:avoid} +#toc,.sidebarblock,.exampleblock>.content{background:none!important} +#toc{border-bottom:1px solid #ddddd8!important;padding-bottom:0!important} +.sect1{padding-bottom:0!important} +.sect1+.sect1{border:0!important} +#header>h1:first-child{margin-top:1.25rem} +body.book #header{text-align:center} +body.book #header>h1:first-child{border:0!important;margin:2.5em 0 1em 0} +body.book #header .details{border:0!important;display:block;padding:0!important} +body.book #header .details span:first-child{margin-left:0!important} +body.book #header .details br{display:block} +body.book #header .details br+span:before{content:none!important} +body.book #toc{border:0!important;text-align:left!important;padding:0!important;margin:0!important} +body.book #toc,body.book #preamble,body.book h1.sect0,body.book .sect1>h2{page-break-before:always} +.listingblock code[data-lang]:before{display:block} +#footer{background:none!important;padding:0 .9375em} +#footer-text{color:rgba(0,0,0,.6)!important;font-size:.9em} +.hide-on-print{display:none!important} +.print-only{display:block!important} +.hide-for-print{display:none!important} +.show-for-print{display:inherit!important}} +</style> +<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.2.0/css/font-awesome.min.css";> +<script> +(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ +(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), +m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) +})(window,document,'script','//www.google-analytics.com/analytics.js','ga'); +ga('create', 'UA-57264262-1', 'auto'); +ga('send', 'pageview'); +</script> +</head> +<body class="article"> +<div id="header"> +<h1>Apache NiFi Overview</h1> +<div class="details"> +<span id="author" class="author">Apache NiFi Team</span><br> +<span id="email" class="email"><a href="mailto:[email protected]";>[email protected]</a></span><br> +</div> +<div id="toc" class="toc"> +<div id="toctitle">Table of Contents</div> +<ul class="sectlevel1"> +<li><a href="#what-is-apache-nifi">What is Apache NiFi?</a></li> +<li><a href="#the-core-concepts-of-nifi">The core concepts of NiFi</a></li> +<li><a href="#nifi-architecture">NiFi Architecture</a></li> +<li><a href="#performance-expectations-and-characteristics-of-nifi">Performance Expectations and Characteristics of NiFi</a></li> +<li><a href="#high-level-overview-of-key-nifi-features">High Level Overview of Key NiFi Features</a></li> +<li><a href="#references">References</a></li> +</ul> +</div> +</div> +<div id="content"> +<div class="sect1"> +<h2 id="what-is-apache-nifi"><a class="anchor" href="#what-is-apache-nifi"></a>What is Apache NiFi?</h2> +<div class="sectionbody"> +<div class="paragraph"> +<p>Put simply NiFi was built to automate the flow of data between systems. While +the term <em>dataflow</em> is used in a variety of contexts, we’ll use it here +to mean the automated and managed flow of information between systems. This +problem space has been around ever since enterprises had more than one system, +where some of the systems created data and some of the systems consumed data. +The problems and solution patterns that emerged have been discussed and +articulated extensively. A comprehensive and readily consumed form is found in +the <em>Enterprise Integration Patterns</em> <a href="#eip">[eip]</a>.</p> +</div> +<div class="paragraph"> +<p>Some of the high-level challenges of dataflow include:</p> +</div> +<div class="dlist"> +<dl> +<dt class="hdlist1">Systems fail</dt> +<dd> +<p>Networks fail, disks fail, software crashes, people make mistakes.</p> +</dd> +<dt class="hdlist1">Data access exceeds capacity to consume</dt> +<dd> +<p>Sometimes a given data source can outpace some part of the processing or delivery chain - it only takes one weak-link to have an issue.</p> +</dd> +<dt class="hdlist1">Boundary conditions are mere suggestions</dt> +<dd> +<p>You will invariably get data that is too big, too small, too fast, too slow, corrupt, wrong, or in the wrong format.</p> +</dd> +<dt class="hdlist1">What is noise one day becomes signal the next</dt> +<dd> +<p>Priorities of an organization change - rapidly. Enabling new flows and changing existing ones must be fast.</p> +</dd> +<dt class="hdlist1">Systems evolve at different rates</dt> +<dd> +<p>The protocols and formats used by a given system can change anytime and often irrespective of the systems around them. Dataflow exists to connect what is essentially a massively distributed system of components that are loosely or not-at-all designed to work together.</p> +</dd> +<dt class="hdlist1">Compliance and security</dt> +<dd> +<p>Laws, regulations, and policies change. Business to business agreements change. System to system and system to user interactions must be secure, trusted, accountable.</p> +</dd> +<dt class="hdlist1">Continuous improvement occurs in production</dt> +<dd> +<p>It is often not possible to come even close to replicating production environments in the lab.</p> +</dd> +</dl> +</div> +<div class="paragraph"> +<p>Over the years dataflow has been one of those necessary evils in an +architecture. Now though there are a number of active and rapidly evolving +movements making dataflow a lot more interesting and a lot more vital to the +success of a given enterprise. These include things like; Service Oriented +Architecture <a href="#soa">[soa]</a>, the rise of the API <a href="#api">[api]</a><a href="#api2">[api2]</a>, Internet of Things <a href="#iot">[iot]</a>, +and Big Data <a href="#bigdata">[bigdata]</a>. In addition, the level of rigor necessary for +compliance, privacy, and security is constantly on the rise. Even still with +all of these new concepts coming about, the patterns and needs of dataflow are +still largely the same. The primary differences then are the scope of +complexity, the rate of change necessary to adapt, and that at scale +the edge case becomes common occurrence. NiFi is built to help tackle these +modern dataflow challenges.</p> +</div> +</div> +</div> +<div class="sect1"> +<h2 id="the-core-concepts-of-nifi"><a class="anchor" href="#the-core-concepts-of-nifi"></a>The core concepts of NiFi</h2> +<div class="sectionbody"> +<div class="paragraph"> +<p>NiFi’s fundamental design concepts closely relate to the main ideas of Flow Based +Programming <a href="#fbp">[fbp]</a>. Here are some of +the main NiFi concepts and how they map to FBP:</p> +</div> +<table class="tableblock frame-all grid-rows spread"> +<colgroup> +<col style="width: 18%;"> +<col style="width: 18%;"> +<col style="width: 62%;"> +</colgroup> +<thead> +<tr> +<th class="tableblock halign-left valign-top">NiFi Term</th> +<th class="tableblock halign-left valign-top">FBP Term</th> +<th class="tableblock halign-left valign-top">Description</th> +</tr> +</thead> +<tbody> +<tr> +<td class="tableblock halign-left valign-top"><p class="tableblock">FlowFile</p></td> +<td class="tableblock halign-left valign-top"><p class="tableblock">Information Packet</p></td> +<td class="tableblock halign-left valign-top"><p class="tableblock">A FlowFile represents each object moving through the system and for each one, NiFi +keeps track of a map of key/value pair attribute strings and its associated +content of zero or more bytes.</p></td> +</tr> +<tr> +<td class="tableblock halign-left valign-top"><p class="tableblock">FlowFile Processor</p></td> +<td class="tableblock halign-left valign-top"><p class="tableblock">Black Box</p></td> +<td class="tableblock halign-left valign-top"><p class="tableblock">Processors actually perform the work. In <a href="#eip">[eip]</a> terms a processor is +doing some combination of data Routing, Transformation, or Mediation between +systems. Processors have access to attributes of a given FlowFile and its +content stream. Processors can operate on zero or more FlowFiles in a given unit of work +and either commit that work or rollback.</p></td> +</tr> +<tr> +<td class="tableblock halign-left valign-top"><p class="tableblock">Connection</p></td> +<td class="tableblock halign-left valign-top"><p class="tableblock">Bounded Buffer</p></td> +<td class="tableblock halign-left valign-top"><p class="tableblock">Connections provide the actual linkage between processors. These act as queues +and allow various processes to interact at differing rates. These queues then +can be prioritized dynamically and can have upper bounds on load, which enable +back pressure.</p></td> +</tr> +<tr> +<td class="tableblock halign-left valign-top"><p class="tableblock">Flow Controller</p></td> +<td class="tableblock halign-left valign-top"><p class="tableblock">Scheduler</p></td> +<td class="tableblock halign-left valign-top"><p class="tableblock">The Flow Controller maintains the knowledge of how processes actually connect +and manages the threads and allocations thereof which all processes use. The +Flow Controller acts as the broker facilitating the exchange of FlowFiles +between processors.</p></td> +</tr> +<tr> +<td class="tableblock halign-left valign-top"><p class="tableblock">Process Group</p></td> +<td class="tableblock halign-left valign-top"><p class="tableblock">subnet</p></td> +<td class="tableblock halign-left valign-top"><p class="tableblock">A Process Group is a specific set of processes and their connections, which can +receive data via input ports and send data out via output ports. In +this manner process groups allow creation of entirely new components simply by +composition of other components.</p></td> +</tr> +</tbody> +</table> +<div class="paragraph"> +<p>This design model, also similar to <a href="#seda">[seda]</a>, provides many beneficial consequences that help NiFi +to be a very effective platform for building powerful and scalable dataflows. +A few of these benefits include:</p> +</div> +<div class="ulist"> +<ul> +<li> +<p>Lends well to visual creation and management of directed graphs of processors</p> +</li> +<li> +<p>Is inherently asynchronous which allows for very high throughput and natural buffering even as processing and flow rates fluctuate</p> +</li> +<li> +<p>Provides a highly concurrent model without a developer having to worry about the typical complexities of concurrency</p> +</li> +<li> +<p>Promotes the development of cohesive and loosely coupled components which can then be reused in other contexts and promotes testable units</p> +</li> +<li> +<p>The resource constrained connections make critical functions such as back-pressure and pressure release very natural and intuitive</p> +</li> +<li> +<p>Error handling becomes as natural as the happy-path rather than a coarse grained catch-all</p> +</li> +<li> +<p>The points at which data enters and exits the system as well as how it flows through are well understood and easily tracked</p> +</li> +</ul> +</div> +</div> +</div> +<div class="sect1"> +<h2 id="nifi-architecture"><a class="anchor" href="#nifi-architecture"></a>NiFi Architecture</h2> +<div class="sectionbody"> +<div class="imageblock"> +<div class="content"> +<img src="./images/nifi-arch.png" alt="NiFi Architecture Diagram"> +</div> +</div> +<div class="paragraph"> +<p>NiFi executes within a JVM living within a host operating system. The primary +components of NiFi then living within the JVM are as follows:</p> +</div> +<div class="dlist"> +<dl> +<dt class="hdlist1">Web Server</dt> +<dd> +<p>The purpose of the web server is to host NiFi’s HTTP-based command and control API.</p> +</dd> +<dt class="hdlist1">Flow Controller</dt> +<dd> +<p>The flow controller is the brains of the operation. It provides threads for extensions to run on and manages their schedule of when they’ll receive resources to execute.</p> +</dd> +<dt class="hdlist1">Extensions</dt> +<dd> +<p>There are various types of extensions for NiFi which will be described in other documents. But the key point here is that extensions operate/execute within the JVM.</p> +</dd> +<dt class="hdlist1">FlowFile Repository</dt> +<dd> +<p>The FlowFile Repository is where NiFi keeps track of the state of what it knows about a given FlowFile that is presently active in the flow. The implementation of the repository is pluggable. The default approach is a persistent Write-Ahead Log that lives on a specified disk partition.</p> +</dd> +<dt class="hdlist1">Content Repository</dt> +<dd> +<p>The Content Repository is where the actual content bytes of a given FlowFile live. The implementation of the repository is pluggable. The default approach is a fairly simple mechanism, which stores blocks of data in the file system. More than one file system storage location can be specified so as to get different physical partitions engaged to reduce contention on any single volume.</p> +</dd> +<dt class="hdlist1">Provenance Repository</dt> +<dd> +<p>The Provenance Repository is where all provenance event data is stored. The repository construct is pluggable with the default implementation being to use one or more physical disk volumes. Within each location event data is indexed and searchable.</p> +</dd> +</dl> +</div> +<div class="paragraph"> +<p>NiFi is also able to operate within a cluster.</p> +</div> +<div class="imageblock"> +<div class="content"> +<img src="./images/nifi-arch-cluster.png" alt="NiFi Cluster Architecture Diagram"> +</div> +</div> +<div class="paragraph"> +<p>A NiFi cluster is comprised of one or more <em>NiFi Nodes</em> (Node) controlled +by a single NiFi Cluster Manager (NCM). The design of clustering is a simple +master/slave model where the NCM is the master and the Nodes are the slaves. +The NCM’s reason for existence is to keep track of which Nodes are in the cluster, +their status, and to replicate requests to modify or observe the +flow. Fundamentally, then, the NCM keeps the state of the cluster consistent. +While the model is that of master and slave, if the master dies the Nodes are all +instructed to continue operating as they were to ensure the data flow remains live. +The absence of the NCM simply means new nodes cannot join the cluster and cluster flow changes +cannot occur until the NCM is restored.</p> +</div> +</div> +</div> +<div class="sect1"> +<h2 id="performance-expectations-and-characteristics-of-nifi"><a class="anchor" href="#performance-expectations-and-characteristics-of-nifi"></a>Performance Expectations and Characteristics of NiFi</h2> +<div class="sectionbody"> +<div class="paragraph"> +<p>NiFi is designed to fully leverage the capabilities of the underlying host system +it is operating on. This maximization of resources is particularly strong with +regard to CPU and disk. Many more details will +be provided on best practices and configuration tips in the Administration Guide.</p> +</div> +<div class="dlist"> +<dl> +<dt class="hdlist1">For IO</dt> +<dd> +<p>The throughput or latency +one can expect to see will vary greatly on how the system is configured. Given +that there are pluggable approaches to most of the major NiFi subsystems the +performance will depend on the implementation. But, for something concrete and broadly +applicable, let’s consider the out-of-the-box default implementations that are used. +These are all persistent with guaranteed delivery and do so using local disk. So +being conservative, assume roughly 50 MB/s read/write rate on modest disks or RAID volumes +within a typical server. NiFi for a large class of dataflows then should be able to +efficiently reach 100 or more MB/s of throughput. That is because linear growth +is expected for each physical partition and content repository added to NiFi. This will +bottleneck at some point on the FlowFile repository and provenance repository. +We plan to provide a benchmarking/performance test template to +include in the build, which will allow users to easily test their system and +to identify where bottlenecks are and at which point they might become a factor. It +should also make it easy for system administrators to make changes and to verify the impact.</p> +</dd> +<dt class="hdlist1">For CPU</dt> +<dd> +<p>The Flow Controller acts as the engine dictating when a particular processor will be +given a thread to execute. Processors should be written to return the thread +as soon as they’re done executing their task. The Flow Controller can be given a +configuration value indicating how many threads there should be for the various +thread pools it maintains. The ideal number of threads to use will depend on the +resources of the host system in terms of numbers of cores, whether that system is +running other services as well, and the nature of the processing in the flow. For +typical IO heavy flows though it would be quite reasonable to set many dozens of threads +to be available if not more.</p> +</dd> +<dt class="hdlist1">For RAM</dt> +<dd> +<p>NiFi lives within the JVM and is thus generally limited to the memory space it +is afforded by the JVM. Garbage collection of the JVM becomes a very important +factor to both restricting the total practical size the heap can be as well as +how well the application will run over time.</p> +</dd> +</dl> +</div> +</div> +</div> +<div class="sect1"> +<h2 id="high-level-overview-of-key-nifi-features"><a class="anchor" href="#high-level-overview-of-key-nifi-features"></a>High Level Overview of Key NiFi Features</h2> +<div class="sectionbody"> +<div class="dlist"> +<dl> +<dt class="hdlist1">Guaranteed Delivery</dt> +<dd> +<p>A core philosophy of NiFi has been that even at very high scale, guaranteed delivery +is a must. This is achieved through effective use of a purpose-built persistent +write-ahead log and content repository. Together they are designed in such a way +as to allow for very high transaction rates, effective load-spreading, copy-on-write, +and play to the strengths of traditional disk read/writes.</p> +</dd> +<dt class="hdlist1">Data Buffering w/ Back Pressure and Pressure Release</dt> +<dd> +<p>NiFi supports buffering of all queued data as well as the ability to +provide back pressure as those queues reach specified limits or to age off data +as it reaches a specified age (its value has perished).</p> +</dd> +<dt class="hdlist1">Prioritized Queuing</dt> +<dd> +<p>NiFi allows the setting of one or more prioritization schemes for how data is +retrieved from a queue. The default is oldest first, but there are times when +data should be pulled newest first, largest first, or some other custom scheme.</p> +</dd> +<dt class="hdlist1">Flow Specific QoS (latency v throughput, loss tolerance, etc.)</dt> +<dd> +<p>There are points of a dataflow where the data is absolutely critical and it is +loss intolerant. There are also times when it must be processed and delivered within +seconds to be of any value. NiFi enables the fine-grained flow specific configuration +of these concerns.</p> +</dd> +<dt class="hdlist1">Data Provenance</dt> +<dd> +<p>NiFi automatically records, indexes, and makes available provenance data as +objects flow through the system even across fan-in, fan-out, transformations, and +more. This information becomes extremely critical in supporting compliance, +troubleshooting, optimization, and other scenarios.</p> +</dd> +<dt class="hdlist1">Recovery / Recording a rolling buffer of fine-grained history</dt> +<dd> +<p>NiFi’s content repository is designed to act as a rolling buffer of history. Data +is removed only as it ages off the content repository or as space is needed. This +combined with the data provenance capability makes for an incredibly useful basis +to enable click-to-content, download of content, and replay, all at a specific +point in an object’s lifecycle which can even span generations.</p> +</dd> +<dt class="hdlist1">Visual Command and Control</dt> +<dd> +<p>Dataflows can become quite complex. Being able to visualize those flows and express +them visually can help greatly to reduce that complexity and to identify areas that +need to be simplified. NiFi enables not only the visual establishment of dataflows but +it does so in real-time. Rather than being <em>design and deploy</em> it is much more like +molding clay. If you make a change to the dataflow that change immediately takes effect. Changes +are fine-grained and isolated to the affected components. You don’t need to stop an entire +flow or set of flows just to make some specific modification.</p> +</dd> +<dt class="hdlist1">Flow Templates</dt> +<dd> +<p>Dataflows tend to be highly pattern oriented and while there are often many different +ways to solve a problem, it helps greatly to be able to share those best practices. Templates +allow subject matter experts to build and publish their flow designs and for others to benefit +and collaborate on them.</p> +</dd> +<dt class="hdlist1">Security</dt> +<dd> +<div class="dlist"> +<dl> +<dt class="hdlist1">System to system</dt> +<dd> +<p>A dataflow is only as good as it is secure. NiFi at every point in a dataflow offers secure +exchange through the use of protocols with encryption such as 2-way SSL. In addition +NiFi enables the flow to encrypt and decrypt content and use shared-keys or other mechanisms on +either side of the sender/recipient equation.</p> +</dd> +<dt class="hdlist1">User to system</dt> +<dd> +<p>NiFi enables 2-Way SSL authentication and provides pluggable authorization so that it can properly control +a user’s access and at particular levels (read-only, dataflow manager, admin). If a user enters a +sensitive property like a password into the flow, it is immediately encrypted server side and never again exposed +on the client side even in its encrypted form.</p> +</dd> +</dl> +</div> +</dd> +<dt class="hdlist1">Designed for Extension</dt> +<dd> +<p>NiFi is at its core built for extension and as such it is a platform on which dataflow processes can execute and interact in a predictable and repeatable manner.</p> +<div class="dlist"> +<dl> +<dt class="hdlist1">Points of extension</dt> +<dd> +<p>Processors, Controller Services, Reporting Tasks, Prioritizers, Customer User Interfaces</p> +</dd> +<dt class="hdlist1">Classloader Isolation</dt> +<dd> +<p>For any component-based system, dependency nightmares can quickly occur. NiFi addresses this by providing a custom class loader model, +ensuring that each extension bundle is exposed to a very limited set of dependencies. As a result, extensions can be built with little concern for whether +they might conflict with another extension. The concept of these extension bundles is called <em>NiFi Archives</em> and will be discussed in greater detail +in the developer’s guide.</p> +</dd> +</dl> +</div> +</dd> +<dt class="hdlist1">Clustering (scale-out)</dt> +<dd> +<p>NiFi is designed to scale-out through the use of clustering many nodes together as described above. If a single node is provisioned and configured +to handle hundreds of MB/s then a modest cluster could be configured to handle GB/s. This then brings about interesting challenges of load balancing +and fail-over between NiFi and the systems from which it gets data. Use of asynchronous queuing based protocols like messaging services, Kafka, etc., can +help. Use of NiFi’s <em>site-to-site</em> feature is also very effective as it is a protocol that allows NiFi and a client (could be another NiFi cluster) to talk to each other, share information +about loading, and to exchange data on specific authorized ports.</p> +</dd> +</dl> +</div> +</div> +</div> +<div class="sect1"> +<h2 id="references"><a class="anchor" href="#references"></a>References</h2> +<div class="sectionbody"> +<div class="ulist bibliography"> +<ul class="bibliography"> +<li> +<p><a id="eip"></a>[eip] Gregor Hohpe. Enterprise Integration Patterns [online]. Retrieved: 27 Dec 2014, from: <a href="http://www.enterpriseintegrationpatterns.com/"; class="bare">http://www.enterpriseintegrationpatterns.com/</a></p> +</li> +<li> +<p><a id="soa"></a>[soa] Wikipedia. Service Oriented Architecture [online]. Retrieved: 27 Dec 2014, from: <a href="http://en.wikipedia.org/wiki/Service-oriented_architecture"; class="bare">http://en.wikipedia.org/wiki/Service-oriented_architecture</a></p> +</li> +<li> +<p><a id="api"></a>[api] Eric Savitz. Welcome to the API Economy [online]. Forbes.com. Retrieved: 27 Dec 2014, from: <a href="http://www.forbes.com/sites/ciocentral/2012/08/29/welcome-to-the-api-economy/"; class="bare">http://www.forbes.com/sites/ciocentral/2012/08/29/welcome-to-the-api-economy/</a></p> +</li> +<li> +<p><a id="api2"></a>[api2] Adam Duvander. The rise of the API economy and consumer-led ecosystems [online]. thenextweb.com. Retrieved: 27 Dec 2014, from: <a href="http://thenextweb.com/dd/2014/03/28/api-economy/"; class="bare">http://thenextweb.com/dd/2014/03/28/api-economy/</a></p> +</li> +<li> +<p><a id="iot"></a>[iot] Wikipedia. Internet of Things [online]. Retrieved: 27 Dec 2014, from: <a href="http://en.wikipedia.org/wiki/Internet_of_Things"; class="bare">http://en.wikipedia.org/wiki/Internet_of_Things</a></p> +</li> +<li> +<p><a id="bigdata"></a>[bigdata] Wikipedia. Big Data [online]. Retrieved: 27 Dec 2014, from: <a href="http://en.wikipedia.org/wiki/Big_data"; class="bare">http://en.wikipedia.org/wiki/Big_data</a></p> +</li> +<li> +<p><a id="fbp"></a>[fbp] Wikipedia. Flow Based Programming [online]. Retrieved: 28 Dec 2014, from: <a href="http://en.wikipedia.org/wiki/Flow-based_programming#Concepts"; class="bare">http://en.wikipedia.org/wiki/Flow-based_programming#Concepts</a></p> +</li> +<li> +<p><a id="seda"></a>[seda] Matt Welsh. Harvard. SEDA: An Architecture for Highly Concurrent Server Applications [online]. Retrieved: 28 Dec 2014, from: <a href="http://www.eecs.harvard.edu/~mdw/proj/seda/"; class="bare">http://www.eecs.harvard.edu/~mdw/proj/seda/</a></p> +</li> +</ul> +</div> +</div> +</div> +</div> +<div id="footer"> +<div id="footer-text"> +Last updated 2015-05-12 22:47:10 EDT +</div> +</div> +</body> +</html> \ No newline at end of file
Added: incubator/nifi/site/trunk/docs/rest-api/images/bgNifiLogo.png URL: http://svn.apache.org/viewvc/incubator/nifi/site/trunk/docs/rest-api/images/bgNifiLogo.png?rev=1679130&view=auto ============================================================================== Binary file - no diff available. Propchange: incubator/nifi/site/trunk/docs/rest-api/images/bgNifiLogo.png ------------------------------------------------------------------------------ svn:mime-type = application/octet-stream Added: incubator/nifi/site/trunk/docs/rest-api/images/nifi16.ico URL: http://svn.apache.org/viewvc/incubator/nifi/site/trunk/docs/rest-api/images/nifi16.ico?rev=1679130&view=auto ============================================================================== Binary file - no diff available. Propchange: incubator/nifi/site/trunk/docs/rest-api/images/nifi16.ico ------------------------------------------------------------------------------ svn:mime-type = application/octet-stream
