Modified: flink/site/docs/0.6-incubating/local_setup.html URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/local_setup.html?rev=1657551&r1=1657550&r2=1657551&view=diff ============================================================================== --- flink/site/docs/0.6-incubating/local_setup.html (original) +++ flink/site/docs/0.6-incubating/local_setup.html Thu Feb 5 12:21:38 2015 @@ -5,154 +5,191 @@ <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1"> <title>Apache Flink (incubating): Local Setup</title> - <link rel="stylesheet" href="/css/bootstrap.css"> - <link rel="stylesheet" href="/css/bootstrap-lumen-custom.css"> - <link rel="stylesheet" href="/css/syntax.css"> - <link rel="/css/custom.css"> - <link rel="css/codetabs.css"> - <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.1.0/css/font-awesome.min.css" rel="stylesheet"> + <link rel="shortcut icon" href="favicon.ico" type="image/x-icon"> + <link rel="icon" href="favicon.ico" type="image/x-icon"> + <link rel="stylesheet" href="css/bootstrap.css"> + <link rel="stylesheet" href="css/bootstrap-lumen-custom.css"> + <link rel="stylesheet" href="css/syntax.css"> + <link rel="stylesheet" href="css/custom.css"> + <link href="css/main/main.css" rel="stylesheet"> <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js";></script> - <script src="/js/bootstrap.min.js"></script> + <script src="js/bootstrap.min.js"></script> <script src="js/codetabs.js"></script> </head> <body> <nav class="navbar navbar-default navbar-fixed-top" role="navigation"> <div class="container"> - <div class="navbar-header"> - <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> - <span class="sr-only">Toggle navigation</span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <a class="navbar-brand" href="http://flink.incubator.apache.org";>Apache Flink</a> - </div> - - <div class="collapse navbar-collapse" id="navbar-collapse-1"> - <ul class="nav navbar-nav"> + <div class="row"> + <div class="col-md-1 af-mobile-nav-bar"> + <a href="index.html" title="Home"> + <img class="hidden-xs hidden-sm img-responsive" + src="img/logo.png" alt="Apache Flink Logo"> + </a> + <div class="row visible-xs"> + <div class="col-xs-3"> + <a href="index.html" title="Home"> + <img class="hidden-x hidden-sm img-responsive" + src="img/logo.png" alt="Apache Flink Logo"> + </a> + </div> + <div class="col-xs-5"></div> + <div class="col-xs-4"> + <div class="af-mobile-btn"> + <span class="glyphicon glyphicon-plus"></span> + </div> + </div> + </div> + </div> + <!-- Navigation --> + <div class="col-md-11"> + <div class="collapse navbar-collapse" id="navbar-collapse-1"> + <ul class="nav navbar-nav"> + + <li> + <a href="index.html" class="">Documentation</a> + </li> + + <li> + <a href="api/java/index.html">Javadoc</a> + </li> + + <li> + <a href="api/scala/index.html#org.apache.flink.api.scala.package">Scaladoc</a> + </li> - <li> - <a href="index.html" class="">Documentation</a> - </li> - - <li> - <a href="api/java/index.html">Javadoc</a> - </li> - </ul> + </ul> + </div> + </div> </div> </div> </nav> - <div style="padding-top:70px" class="container"> + + <div style="padding-top:120px" class="container"> <div class="row"> <div class="col-md-3"> <ul> - <li>Quickstart - <ul> - <li><a href="setup_quickstart.html">Install</a></li> - <li><a href="run_example_quickstart.html">Run Example</a></li> - <li><a href="java_api_quickstart.html">Java API</a></li> - <li><a href="scala_api_quickstart.html">Scala API</a></li> - <li><a href="faq.html">FAQ</a></li> - </ul> - </li> - - <li>Setup & Configuration - <ul> - <li><a href="building.html">Build Instructions</a></li> - <li><a href="local_setup.html">Local Setup</a></li> - <li><a href="cluster_setup.html">Cluster Setup</a></li> - <li><a href="yarn_setup.html">YARN Setup</a></li> - <li><a href="config.html">Configuration</a></li> - </ul> - </li> - - <li>Programming Guides - <ul> - <li><a href="java_api_guide.html">Java API</a></li> - <li><a href="java_api_transformations.html">Java API Transformations</a></li> - <li><a href="scala_api_guide.html">Scala API</a></li> - <li><a href="iterations.html">Iterations</a></li> - <li><a href="spargel_guide.html">Spargel Graph API</a></li> - </ul> - </li> - - <li>Examples - <ul> - <li><a href="java_api_examples.html">Java API</a></li> - <li><a href="scala_api_examples.html">Scala API</a></li> - <li><a href="example_connectors.html">Connecting to other systems</a></li> - </ul> - </li> - - <li>Execution - <ul> - <li><a href="local_execution.html">Local/Debugging</a></li> - <li><a href="cluster_execution.html">Cluster</a></li> - <li><a href="cli.html">Command-Line Interface</a></li> - <li><a href="web_client.html">Web Interface</a></li> - </ul> - </li> - - <li>Internals - <ul> - <li><a href="internal_overview.html">Overview</a></li> - <li><a href="internal_general_arch.html">General Architecture</a></li> - <li><a href="internal_add_operator.html">How-to: Adding a new Operator</a></li> - </ul> - </li> + <li><a href="faq.html">FAQ</a></li> + <li>Quickstart + <ul> + <li><a href="setup_quickstart.html">Setup</a></li> + <li><a href="run_example_quickstart.html">Run Example</a></li> + <li><a href="java_api_quickstart.html">Java API</a></li> + <li><a href="scala_api_quickstart.html">Scala API</a></li> + </ul> + </li> + + <li>Setup & Configuration + <ul> + <li><a href="local_setup.html">Local Setup</a></li> + <li><a href="building.html">Build Flink</a></li> + <li><a href="cluster_setup.html">Cluster Setup</a></li> + <li><a href="yarn_setup.html">YARN Setup</a></li> + <li><a href="config.html">Configuration</a></li> + </ul> + </li> + + <li>Programming Guides + <ul> + <li><a href="programming_guide.html">Programming Guide</a></li> + <li><a href="dataset_transformations.html">DataSet Transformations</a></li> + <li><a href="java8_programming_guide.html">Java 8 Programming Guide</a></li> + <li><a href="streaming_guide.html">Streaming Guide</a></li> + <li><a href="iterations.html">Iterations</a></li> + <li><a href="spargel_guide.html">Spargel Graph API</a></li> + <li><a href="hadoop_compatibility.html">Hadoop Compatibility</a></li> + </ul> + </li> + + <li>Examples + <ul> + <li><a href="examples.html">Bundled Examples</a></li> + <li><a href="example_connectors.html">Connecting to other systems</a></li> + </ul> + </li> + + <li>Execution + <ul> + <li><a href="local_execution.html">Local/Debugging</a></li> + <li><a href="cluster_execution.html">Cluster</a></li> + <li><a href="cli.html">Command-Line Interface</a></li> + <li><a href="web_client.html">Web Interface</a></li> + </ul> + </li> + + <li>Internals + <ul> + <li><a href="internal_overview.html">Overview</a></li> + </ul> + </li> </ul> + </div> <div class="col-md-9"> <h1>Local Setup</h1> + + <ul id="markdown-toc"> + <li><a href="#download">Download</a></li> + <li><a href="#requirements">Requirements</a></li> + <li><a href="#configuration">Configuration</a></li> + <li><a href="#starting-flink">Starting Flink</a></li> + <li><a href="#flink-on-windows">Flink on Windows</a> <ul> + <li><a href="#starting-with-windows-batch-files">Starting with Windows Batch Files</a></li> + <li><a href="#starting-with-cygwin-and-unix-scripts">Starting with Cygwin and Unix Scripts</a></li> + <li><a href="#installing-flink-from-git">Installing Flink from Git</a></li> + </ul> + </li> +</ul> - <p>This documentation is intended to provide instructions on how to run Flink locally on a single machine.</p> +<p>This documentation is intended to provide instructions on how to run Flink locally on a single machine.</p> -<h1 id="download">Download</h1> +<h2 id="download">Download</h2> -<p>Go to the <a href="/downloads/">downloads page</a> and get the ready to run package. If you want to interact with Hadoop (e.g. HDFS or HBase), make sure to pick the Flink package <strong>matching your Hadoop version</strong>. When in doubt or you plan to just work with the local file system pick the package for Hadoop 1.2.x.</p> +<p>Go to the <a href="/downloads.html">downloads page</a> and get the ready to run package. If you want to interact with Hadoop (e.g. HDFS or HBase), make sure to pick the Flink package <strong>matching your Hadoop version</strong>. When in doubt or you plan to just work with the local file system pick the package for Hadoop 1.2.x.</p> -<h1 id="requirements">Requirements</h1> +<h2 id="requirements">Requirements</h2> -<p>Flink runs on <strong>Linux</strong>, <strong>Mac OS X</strong> and <strong>Windows</strong>. The only requirement for a local setup is <strong>Java 1.6.x</strong> or higher. The following manual assumes a <em>UNIX-like environment</em>, for Windows see <a href="#windows">Flink on Windows</a>.</p> +<p>Flink runs on <strong>Linux</strong>, <strong>Mac OS X</strong> and <strong>Windows</strong>. The only requirement for a local setup is <strong>Java 1.6.x</strong> or higher. The following manual assumes a <em>UNIX-like environment</em>, for Windows see <a href="#flink-on-windows">Flink on Windows</a>.</p> <p>You can check the correct installation of Java by issuing the following command:</p> -<div class="highlight"><pre><code class="language-bash" data-lang="bash">java -version -</code></pre></div> + +<div class="highlight"><pre><code class="language-bash">java -version</code></pre></div> + <p>The command should output something comparable to the following:</p> -<div class="highlight"><pre><code class="language-bash" data-lang="bash">java version <span class="s2">"1.6.0_22"</span> + +<div class="highlight"><pre><code class="language-bash">java version <span class="s2">"1.6.0_22"</span> Java<span class="o">(</span>TM<span class="o">)</span> SE Runtime Environment <span class="o">(</span>build 1.6.0_22-b04<span class="o">)</span> -Java HotSpot<span class="o">(</span>TM<span class="o">)</span> 64-Bit Server VM <span class="o">(</span>build 17.1-b03, mixed mode<span class="o">)</span> -</code></pre></div> -<h1 id="configuration">Configuration</h1> +Java HotSpot<span class="o">(</span>TM<span class="o">)</span> 64-Bit Server VM <span class="o">(</span>build 17.1-b03, mixed mode<span class="o">)</span></code></pre></div> + +<h2 id="configuration">Configuration</h2> -<p><strong>For local mode Flink is ready to go out of the box and you don't need to change the default configuration.</strong></p> +<p><strong>For local mode Flink is ready to go out of the box and you donât need to change the default configuration.</strong></p> <p>The out of the box configuration will use your default Java installation. You can manually set the environment variable <code>JAVA_HOME</code> or the configuration key <code>env.java.home</code> in <code>conf/flink-conf.yaml</code> if you want to manually override the Java runtime to use. Consult the <a href="config.html">configuration page</a> for further details about configuring Flink.</p> -<h1 id="starting-flink">Starting Flink</h1> +<h2 id="starting-flink">Starting Flink</h2> <p><strong>You are now ready to start Flink.</strong> Unpack the downloaded archive and change to the newly created <code>flink</code> directory. There you can start Flink in local mode:</p> -<div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nv">$ </span>tar xzf flink-*.tgz + +<div class="highlight"><pre><code class="language-bash"><span class="nv">$ </span>tar xzf flink-*.tgz <span class="nv">$ </span><span class="nb">cd </span>flink <span class="nv">$ </span>bin/start-local.sh -Starting job manager -</code></pre></div> +Starting job manager</code></pre></div> + <p>You can check that the system is running by checking the log files in the <code>logs</code> directory:</p> -<div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nv">$ </span>tail log/flink-*-jobmanager-*.log + +<div class="highlight"><pre><code class="language-bash"><span class="nv">$ </span>tail log/flink-*-jobmanager-*.log INFO ... - Initializing memory manager with <span class="m">409</span> megabytes of memory INFO ... - Trying to load org.apache.flinknephele.jobmanager.scheduler.local.LocalScheduler as scheduler INFO ... - Setting up web info server, using web-root directory ... INFO ... - Web info server will display information about nephele job-manager on localhost, port 8081. -INFO ... - Starting web info server <span class="k">for</span> JobManager on port 8081 -</code></pre></div> -<p>The JobManager will also start a web frontend on port 8081, which you can check with your browser at <code>http://localhost:8081</code>.</p> +INFO ... - Starting web info server <span class="k">for</span> JobManager on port 8081</code></pre></div> -<p><section id="windows"></p> +<p>The JobManager will also start a web frontend on port 8081, which you can check with your browser at <code>http://localhost:8081</code>.</p> -<h1 id="flink-on-windows">Flink on Windows</h1> +<h2 id="flink-on-windows">Flink on Windows</h2> <p>If you want to run Flink on Windows you need to download, unpack and configure the Flink archive as mentioned above. After that you can either use the <strong>Windows Batch</strong> file (<code>.bat</code>) or use <strong>Cygwin</strong> to run the Flink Jobmanager.</p> @@ -160,46 +197,56 @@ INFO ... - Starting web info server <spa <p>To start Flink in local mode from the <em>Windows Batch</em>, open the command window, navigate to the <code>bin/</code> directory of Flink and run <code>start-local.bat</code>.</p> -<p>Note: The <code>bin</code> folder of your Java Runtime Environment must be included in Window's <code>%PATH%</code> variable. Follow this <a href="http://www.java.com/en/download/help/path.xml";>guide</a> to add Java to the <code>%PATH%</code> variable.</p> -<div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nv">$ </span><span class="nb">cd </span>flink +<p>Note: The <code>bin</code> folder of your Java Runtime Environment must be included in Windowâs <code>%PATH%</code> variable. Follow this <a href="http://www.java.com/en/download/help/path.xml";>guide</a> to add Java to the <code>%PATH%</code> variable.</p> + +<div class="highlight"><pre><code class="language-bash"><span class="nv">$ </span><span class="nb">cd </span>flink <span class="nv">$ </span><span class="nb">cd </span>bin <span class="nv">$ </span>start-local.bat Starting Flink job manager. Webinterface by default on http://localhost:8081/. -Do not close this batch window. Stop job manager by pressing Ctrl+C. -</code></pre></div> +Do not close this batch window. Stop job manager by pressing Ctrl+C.</code></pre></div> + <p>After that, you need to open a second terminal to run jobs using <code>flink.bat</code>.</p> <h3 id="starting-with-cygwin-and-unix-scripts">Starting with Cygwin and Unix Scripts</h3> <p>With <em>Cygwin</em> you need to start the Cygwin Terminal, navigate to your Flink directory and run the <code>start-local.sh</code> script:</p> -<div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nv">$ </span><span class="nb">cd </span>flink + +<div class="highlight"><pre><code class="language-bash"><span class="nv">$ </span><span class="nb">cd </span>flink <span class="nv">$ </span>bin/start-local.sh -Starting Nephele job manager -</code></pre></div> +Starting Nephele job manager</code></pre></div> + <h3 id="installing-flink-from-git">Installing Flink from Git</h3> <p>If you are installing Flink from the git repository and you are using the Windows git shell, Cygwin can produce a failure similiar to this one:</p> -<div class="highlight"><pre><code class="language-bash" data-lang="bash">c:/flink/bin/start-local.sh: line 30: <span class="s1">$'\r'</span>: <span class="nb">command </span>not found -</code></pre></div> + +<div class="highlight"><pre><code class="language-bash">c:/flink/bin/start-local.sh: line 30: <span class="s1">$'\r'</span>: <span class="nb">command </span>not found</code></pre></div> + <p>This error occurs, because git is automatically transforming UNIX line endings to Windows style line endings when running in Windows. The problem is, that Cygwin can only deal with UNIX style line endings. The solution is to adjust the Cygwin settings to deal with the correct line endings by following these three steps:</p> <ol> -<li><p>Start a Cygwin shell.</p></li> -<li><p>Determine your home directory by entering</p></li> + <li> + <p>Start a Cygwin shell.</p> + </li> + <li> + <p>Determine your home directory by entering</p> + </li> </ol> -<div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nb">cd</span><span class="p">;</span> <span class="nb">pwd</span> -</code></pre></div> + +<div class="highlight"><pre><code class="language-bash"><span class="nb">cd</span><span class="p">;</span> <span class="nb">pwd</span></code></pre></div> + <p>It will return a path under the Cygwin root path.</p> <ol> -<li> Using NotePad, WordPad or a different text editor open the file <code>.bash_profile</code> in the home directory and append the following: (If the file does not exist you have to create it)</li> + <li>Using NotePad, WordPad or a different text editor open the file <code>.bash_profile</code> in the home directory and append the following: (If the file does not exist you have to create it)</li> </ol> -<div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nb">export </span>SHELLOPTS -<span class="nb">set</span> -o igncr -</code></pre></div> -<p>Save the file and open a new bash shell.</p> +<div class="highlight"><pre><code class="language-bash"><span class="nb">export </span>SHELLOPTS +<span class="nb">set</span> -o igncr</code></pre></div> +<p>Save the file and open a new bash shell.</p> + + + <!-- Disqus Area --> <div style="padding-top:30px" id="disqus_thread"></div> <script type="text/javascript"> @@ -214,14 +261,12 @@ Starting Nephele job manager })(); </script> <noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript";>comments powered by Disqus.</a></noscript> - <a href="http://disqus.com"; class="dsq-brlink">comments powered by <span class="logo-disqus">Disqus</span></a> - </div> </div> <div class="footer"> - <p><hr class="divider"></p> + <hr class="divider" /> <p><small>Apache Flink is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC. Incubation is @@ -232,9 +277,10 @@ incubation status is not necessarily a r stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.</small></p> -<p><a href="http://incubator.apache.org/";><img src="/img/apache-incubator-logo.png" alt="Incubator Logo"></a></p> +<p><a href="http://incubator.apache.org/";><img src="/img/apache-incubator-logo.png" alt="Incubator Logo" /></a></p> -<p class="text-center"><a href="/privacy-policy.html">Privacy Policy<a></p> +<p class="text-center"><a href="privacy-policy.html">Privacy Policy<a> +</a></a></p> </div> </div>
Modified: flink/site/docs/0.6-incubating/run_example_quickstart.html URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/run_example_quickstart.html?rev=1657551&r1=1657550&r2=1657551&view=diff ============================================================================== --- flink/site/docs/0.6-incubating/run_example_quickstart.html (original) +++ flink/site/docs/0.6-incubating/run_example_quickstart.html Thu Feb 5 12:21:38 2015 @@ -5,243 +5,287 @@ <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1"> <title>Apache Flink (incubating): Quick Start: Run K-Means Example</title> - <link rel="stylesheet" href="/css/bootstrap.css"> - <link rel="stylesheet" href="/css/bootstrap-lumen-custom.css"> - <link rel="stylesheet" href="/css/syntax.css"> - <link rel="/css/custom.css"> - <link rel="css/codetabs.css"> - <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.1.0/css/font-awesome.min.css" rel="stylesheet"> + <link rel="shortcut icon" href="favicon.ico" type="image/x-icon"> + <link rel="icon" href="favicon.ico" type="image/x-icon"> + <link rel="stylesheet" href="css/bootstrap.css"> + <link rel="stylesheet" href="css/bootstrap-lumen-custom.css"> + <link rel="stylesheet" href="css/syntax.css"> + <link rel="stylesheet" href="css/custom.css"> + <link href="css/main/main.css" rel="stylesheet"> <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js";></script> - <script src="/js/bootstrap.min.js"></script> + <script src="js/bootstrap.min.js"></script> <script src="js/codetabs.js"></script> </head> <body> <nav class="navbar navbar-default navbar-fixed-top" role="navigation"> <div class="container"> - <div class="navbar-header"> - <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> - <span class="sr-only">Toggle navigation</span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <a class="navbar-brand" href="http://flink.incubator.apache.org";>Apache Flink</a> - </div> - - <div class="collapse navbar-collapse" id="navbar-collapse-1"> - <ul class="nav navbar-nav"> + <div class="row"> + <div class="col-md-1 af-mobile-nav-bar"> + <a href="index.html" title="Home"> + <img class="hidden-xs hidden-sm img-responsive" + src="img/logo.png" alt="Apache Flink Logo"> + </a> + <div class="row visible-xs"> + <div class="col-xs-3"> + <a href="index.html" title="Home"> + <img class="hidden-x hidden-sm img-responsive" + src="img/logo.png" alt="Apache Flink Logo"> + </a> + </div> + <div class="col-xs-5"></div> + <div class="col-xs-4"> + <div class="af-mobile-btn"> + <span class="glyphicon glyphicon-plus"></span> + </div> + </div> + </div> + </div> + <!-- Navigation --> + <div class="col-md-11"> + <div class="collapse navbar-collapse" id="navbar-collapse-1"> + <ul class="nav navbar-nav"> + + <li> + <a href="index.html" class="">Documentation</a> + </li> + + <li> + <a href="api/java/index.html">Javadoc</a> + </li> + + <li> + <a href="api/scala/index.html#org.apache.flink.api.scala.package">Scaladoc</a> + </li> - <li> - <a href="index.html" class="">Documentation</a> - </li> - - <li> - <a href="api/java/index.html">Javadoc</a> - </li> - </ul> + </ul> + </div> + </div> </div> </div> </nav> - <div style="padding-top:70px" class="container"> + + <div style="padding-top:120px" class="container"> <div class="row"> <div class="col-md-3"> <ul> - <li>Quickstart - <ul> - <li><a href="setup_quickstart.html">Install</a></li> - <li><a href="run_example_quickstart.html">Run Example</a></li> - <li><a href="java_api_quickstart.html">Java API</a></li> - <li><a href="scala_api_quickstart.html">Scala API</a></li> - <li><a href="faq.html">FAQ</a></li> - </ul> - </li> - - <li>Setup & Configuration - <ul> - <li><a href="building.html">Build Instructions</a></li> - <li><a href="local_setup.html">Local Setup</a></li> - <li><a href="cluster_setup.html">Cluster Setup</a></li> - <li><a href="yarn_setup.html">YARN Setup</a></li> - <li><a href="config.html">Configuration</a></li> - </ul> - </li> - - <li>Programming Guides - <ul> - <li><a href="java_api_guide.html">Java API</a></li> - <li><a href="java_api_transformations.html">Java API Transformations</a></li> - <li><a href="scala_api_guide.html">Scala API</a></li> - <li><a href="iterations.html">Iterations</a></li> - <li><a href="spargel_guide.html">Spargel Graph API</a></li> - </ul> - </li> - - <li>Examples - <ul> - <li><a href="java_api_examples.html">Java API</a></li> - <li><a href="scala_api_examples.html">Scala API</a></li> - <li><a href="example_connectors.html">Connecting to other systems</a></li> - </ul> - </li> - - <li>Execution - <ul> - <li><a href="local_execution.html">Local/Debugging</a></li> - <li><a href="cluster_execution.html">Cluster</a></li> - <li><a href="cli.html">Command-Line Interface</a></li> - <li><a href="web_client.html">Web Interface</a></li> - </ul> - </li> - - <li>Internals - <ul> - <li><a href="internal_overview.html">Overview</a></li> - <li><a href="internal_general_arch.html">General Architecture</a></li> - <li><a href="internal_add_operator.html">How-to: Adding a new Operator</a></li> - </ul> - </li> + <li><a href="faq.html">FAQ</a></li> + <li>Quickstart + <ul> + <li><a href="setup_quickstart.html">Setup</a></li> + <li><a href="run_example_quickstart.html">Run Example</a></li> + <li><a href="java_api_quickstart.html">Java API</a></li> + <li><a href="scala_api_quickstart.html">Scala API</a></li> + </ul> + </li> + + <li>Setup & Configuration + <ul> + <li><a href="local_setup.html">Local Setup</a></li> + <li><a href="building.html">Build Flink</a></li> + <li><a href="cluster_setup.html">Cluster Setup</a></li> + <li><a href="yarn_setup.html">YARN Setup</a></li> + <li><a href="config.html">Configuration</a></li> + </ul> + </li> + + <li>Programming Guides + <ul> + <li><a href="programming_guide.html">Programming Guide</a></li> + <li><a href="dataset_transformations.html">DataSet Transformations</a></li> + <li><a href="java8_programming_guide.html">Java 8 Programming Guide</a></li> + <li><a href="streaming_guide.html">Streaming Guide</a></li> + <li><a href="iterations.html">Iterations</a></li> + <li><a href="spargel_guide.html">Spargel Graph API</a></li> + <li><a href="hadoop_compatibility.html">Hadoop Compatibility</a></li> + </ul> + </li> + + <li>Examples + <ul> + <li><a href="examples.html">Bundled Examples</a></li> + <li><a href="example_connectors.html">Connecting to other systems</a></li> + </ul> + </li> + + <li>Execution + <ul> + <li><a href="local_execution.html">Local/Debugging</a></li> + <li><a href="cluster_execution.html">Cluster</a></li> + <li><a href="cli.html">Command-Line Interface</a></li> + <li><a href="web_client.html">Web Interface</a></li> + </ul> + </li> + + <li>Internals + <ul> + <li><a href="internal_overview.html">Overview</a></li> + </ul> + </li> </ul> + </div> <div class="col-md-9"> <h1>Quick Start: Run K-Means Example</h1> + + <ul id="markdown-toc"> + <li><a href="#setup-flink">Setup Flink</a></li> + <li><a href="#generate-input-data">Generate Input Data</a></li> + <li><a href="#inspect-the-input-data">Inspect the Input Data</a></li> + <li><a href="#start-flink">Start Flink</a></li> + <li><a href="#inspect-and-run-the-k-means-example-program">Inspect and Run the K-Means Example Program</a></li> + <li><a href="#shutdown-flink">Shutdown Flink</a></li> + <li><a href="#analyze-the-result">Analyze the Result</a></li> +</ul> - <p>This guide will demonstrate Flink's features by example. You will see how you can leverage Flink's Iteration-feature to find clusters in a dataset using <a href="http://en.wikipedia.org/wiki/K-means_clustering";>K-Means clustering</a>. -On the way, you will see the compiler, the status interface and the result of the algorithm.</p> +<p>This guide walks you through the steps of executing an example program (<a href="http://en.wikipedia.org/wiki/K-means_clustering";>K-Means clustering</a>) on Flink. On the way, you will see the a visualization of the program, the optimized execution plan, and track the progress of its execution.</p> -<h1 id="generate-input-data">Generate Input Data</h1> +<h2 id="setup-flink">Setup Flink</h2> +<p>Follow the <a href="setup_quickstart.html">instructions</a> to setup Flink and enter the root directory of your Flink setup.</p> +<h2 id="generate-input-data">Generate Input Data</h2> <p>Flink contains a data generator for K-Means.</p> -<div class="highlight"><pre><code class="language-bash" data-lang="bash"> <span class="c"># pick a mirror from: http://www.apache.org/dyn/closer.cgi/incubator/flink/flink-0.6.1-incubating/flink-0.6.1-incubating-bin-hadoop1.tgz</span> - <span class="c"># for example</span> -wget http://mirror.lwnetwork.org.uk/APACHE/incubator/flink/flink-0.6-incubating-bin-hadoop1.tgz -tar xzf flink-*.tgz -<span class="nb">cd </span>flink-* + +<div class="highlight"><pre><code class="language-bash"><span class="c"># Assuming you are in the root directory of your Flink setup</span> mkdir kmeans <span class="nb">cd </span>kmeans - <span class="c"># Run data generator</span> -java -cp ../examples/flink-java-examples-0.6.1-incubating-KMeans.jar org.apache.flink.example.java.clustering.util.KMeansDataGenerator <span class="m">500</span> <span class="m">10</span> 0.08 +<span class="c"># Run data generator</span> +java -cp ../examples/flink-java-examples-*-KMeans.jar org.apache.flink.examples.java.clustering.util.KMeansDataGenerator <span class="m">500</span> <span class="m">10</span> 0.08 cp /tmp/points . -cp /tmp/centers . -</code></pre></div> +cp /tmp/centers .</code></pre></div> + <p>The generator has the following arguments:</p> -<div class="highlight"><pre><code class="language-text" data-lang="text">KMeansDataGenerator <numberOfDataPoints> <numberOfClusterCenters> [<relative stddev>] [<centroid range>] [<seed>] -</code></pre></div> -<p>The <em>relative standard deviation</em> is an interesting tuning parameter: it determines the closeness of the points to the centers.</p> -<p>The <code>kmeans/</code> directory should now contain two files: <code>centers</code> and <code>points</code>.</p> +<div class="highlight"><pre><code class="language-bash">KMeansDataGenerator <numberOfDataPoints> <numberOfClusterCenters> <span class="o">[</span><relative stddev><span class="o">]</span> <span class="o">[</span><centroid range><span class="o">]</span> <span class="o">[</span><seed><span class="o">]</span></code></pre></div> + +<p>The <em>relative standard deviation</em> is an interesting tuning parameter. It determines the closeness of the points to randomly generated centers.</p> -<h1 id="review-input-data">Review Input Data</h1> +<p>The <code>kmeans/</code> directory should now contain two files: <code>centers</code> and <code>points</code>. The <code>points</code> file contains the points to cluster and the <code>centers</code> file contains initial cluster centers.</p> + +<h2 id="inspect-the-input-data">Inspect the Input Data</h2> +<p>Use the <code>plotPoints.py</code> tool to review the generated data points. <a href="quickstart/plotPoints.py">Download Python Script</a></p> + +<div class="highlight"><pre><code class="language-bash">python plotPoints.py points ./points input</code></pre></div> -<p>Use the <code>plotPoints.py</code> tool to review the result of the data generator. <a href="quickstart/plotPoints.py">Download Python Script</a></p> -<div class="highlight"><pre><code class="language-bash" data-lang="bash">python plotPoints.py points points input -</code></pre></div> <p>Note: You might have to install <a href="http://matplotlib.org/";>matplotlib</a> (<code>python-matplotlib</code> package on Ubuntu) to use the Python script.</p> <p>You can review the input data stored in the <code>input-plot.pdf</code>, for example with Evince (<code>evince input-plot.pdf</code>).</p> <p>The following overview presents the impact of the different standard deviations on the input data.</p> -<table><thead> -<tr> -<th style="text-align: center">relative stddev = 0.03</th> -<th style="text-align: center">relative stddev = 0.08</th> -<th style="text-align: center">relative stddev = 0.15</th> -</tr> -</thead><tbody> -<tr> -<td style="text-align: center"><img src="img/quickstart-example/kmeans003.png" alt="example1" style="width: 275px;"/></td> -<td style="text-align: center"><img src="img/quickstart-example/kmeans008.png" alt="example2" style="width: 275px;"/></td> -<td style="text-align: center"><img src="img/quickstart-example/kmeans015.png" alt="example3" style="width: 275px;"/></td> -</tr> -</tbody></table> - -<h1 id="run-clustering">Run Clustering</h1> - -<p>We are using the generated input data to run the clustering using a Flink job.</p> -<div class="highlight"><pre><code class="language-bash" data-lang="bash"> <span class="c"># go to the Flink-root directory</span> -<span class="nb">cd </span>flink - <span class="c"># start Flink (use ./bin/start-cluster.sh if you're on a cluster)</span> +<table> + <thead> + <tr> + <th style="text-align: center">relative stddev = 0.03</th> + <th style="text-align: center">relative stddev = 0.08</th> + <th style="text-align: center">relative stddev = 0.15</th> + </tr> + </thead> + <tbody> + <tr> + <td style="text-align: center"><img src="img/quickstart-example/kmeans003.png" alt="example1" style="width: 275px;" /></td> + <td style="text-align: center"><img src="img/quickstart-example/kmeans008.png" alt="example2" style="width: 275px;" /></td> + <td style="text-align: center"><img src="img/quickstart-example/kmeans015.png" alt="example3" style="width: 275px;" /></td> + </tr> + </tbody> +</table> + +<h2 id="start-flink">Start Flink</h2> +<p>Start Flink and the web job submission client on your local machine.</p> + +<div class="highlight"><pre><code class="language-bash"><span class="c"># return to the Flink root directory</span> +<span class="nb">cd</span> .. +<span class="c"># start Flink</span> ./bin/start-local.sh - <span class="c"># Start Flink web client</span> -./bin/start-webclient.sh -</code></pre></div> -<h1 id="review-flink-compiler">Review Flink Compiler</h1> +<span class="c"># Start the web client</span> +./bin/start-webclient.sh</code></pre></div> -<p>The Flink webclient allows to submit Flink programs using a graphical user interface.</p> +<h2 id="inspect-and-run-the-k-means-example-program">Inspect and Run the K-Means Example Program</h2> +<p>The Flink web client allows to submit Flink programs using a graphical user interface.</p> <div class="row" style="padding-top:15px"> - <div class="col-md-6"> - <a data-lightbox="compiler" href="img/quickstart-example/run-webclient.png" data-lightbox="example-1"><img class="img-responsive" src="img/quickstart-example/run-webclient.png" /></a> - </div> - <div class="col-md-6"> - 1. <a href="http://localhost:8080/launch.html";>Open webclient on localhost:8080</a> <br> - 2. Upload the file. - -<div class="highlight"><pre><code class="language-bash" data-lang="bash">examples/flink-java-examples-0.6.1-incubating-KMeans.jar</code></pre></div> - </br> - 3. Select it in the left box to see how the operators in the plan are connected to each other. <br> - 4. Enter the arguments in the lower left box: - -<div class="highlight"><pre><code class="language-bash" data-lang="bash">file://<pathToGenerated>points file://<pathToGenerated>centers file://<pathToGenerated>result 10</code></pre></div> - - For example: - -<div class="highlight"><pre><code class="language-bash" data-lang="bash">file:///tmp/flink/kmeans/points file:///tmp/flink/kmeans/centers file:///tmp/flink/kmeans/result 20</code></pre></div> + <div class="col-md-6"> + <a data-lightbox="example-1" href="img/quickstart-example/run-webclient.png"><img class="img-responsive" src="img/quickstart-example/run-webclient.png" /></a> + </div> + <div class="col-md-6"> + 1. Open web client on <a href="http://localhost:8080/launch.html";>localhost:8080</a> <br /> + 2. Upload the K-Mean job JAR file. + +<div class="highlight"><pre><code class="language-bash" data-lang="bash">./examples/flink-java-examples-*-KMeans.jar</code></pre></div> + + 3. Select it in the left box to see how the operators in the plan are connected to each other. <br /> + 4. Enter the arguments in the lower left box: + +<div class="highlight"><pre><code class="language-bash" data-lang="bash">file://<pathToFlink>/kmeans/points file://<pathToFlink>/kmeans/centers file://<pathToFlink>/kmeans/result 10</code></pre></div> + + For example: + +<div class="highlight"><pre><code class="language-bash" data-lang="bash">file:///tmp/flink/kmeans/points file:///tmp/flink/kmeans/centers file:///tmp/flink/kmeans/result 10</code></pre></div> - </div> + </div> </div> - -<p><hr></p> +<hr /> <div class="row" style="padding-top:15px"> - <div class="col-md-6"> - <a data-lightbox="compiler" href="img/quickstart-example/compiler-webclient-new.png" data-lightbox="example-1"><img class="img-responsive" src="img/quickstart-example/compiler-webclient-new.png" /></a> - </div> - - <div class="col-md-6"> - 1. Press the <b>RunJob</b> to see the optimzer plan. <br> - 2. Inspect the operators and see the properties (input sizes, cost estimation) determined by the optimizer. - </div> + <div class="col-md-6"> + <a data-lightbox="example-1" href="img/quickstart-example/compiler-webclient-new.png"><img class="img-responsive" src="img/quickstart-example/compiler-webclient-new.png" /></a> + </div> + + <div class="col-md-6"> + 1. Press the <b>RunJob</b> to see the optimizer plan. <br /> + 2. Inspect the operators and see the properties (input sizes, cost estimation) determined by the optimizer. + </div> </div> - -<p><hr></p> +<hr /> <div class="row" style="padding-top:15px"> - <div class="col-md-6"> - <a data-lightbox="compiler" href="img/quickstart-example/jobmanager-running-new.png" data-lightbox="example-1"><img class="img-responsive" src="img/quickstart-example/jobmanager-running-new.png" /></a> - </div> - <div class="col-md-6"> - 1. Press the <b>Continue</b> button to start executing the job. <br> - 2. <a href="http://localhost:8080/launch.html";>Open Flink's monitoring interface</a> to see the job's progress.<br> - 3. Once the job has finished, you can analyize the runtime of the individual operators. - </div> + <div class="col-md-6"> + <a data-lightbox="example-1" href="img/quickstart-example/jobmanager-running-new.png"><img class="img-responsive" src="img/quickstart-example/jobmanager-running-new.png" /></a> + </div> + <div class="col-md-6"> + 1. Press the <b>Continue</b> button to start executing the job. <br /> + 2. <a href="http://localhost:8080/launch.html";>Open Flink's monitoring interface</a> to see the job's progress. (Due to the small input data, the job will finish really quick!)<br /> + 3. Once the job has finished, you can analyze the runtime of the individual operators. + </div> </div> -<h1 id="analyze-the-result">Analyze the Result</h1> +<h2 id="shutdown-flink">Shutdown Flink</h2> +<p>Stop Flink when you are done.</p> -<p>Use the <a href="quickstart/plotPoints.py">Python Script</a> again to visualize the result</p> -<div class="highlight"><pre><code class="language-bash" data-lang="bash">python plotPoints.py result result result-pdf -</code></pre></div> -<p>The following three pictures show the results for the sample input above. Play around with the parameters (number of iterations, number of clusters) to see how they affect the result.</p> +<div class="highlight"><pre><code class="language-bash"><span class="c"># stop Flink</span> +./bin/stop-local.sh +<span class="c"># Stop the Flink web client</span> +./bin/stop-webclient.sh</code></pre></div> + +<h2 id="analyze-the-result">Analyze the Result</h2> +<p>Use the <a href="quickstart/plotPoints.py">Python Script</a> again to visualize the result.</p> -<table><thead> -<tr> -<th style="text-align: center">relative stddev = 0.03</th> -<th style="text-align: center">relative stddev = 0.08</th> -<th style="text-align: center">relative stddev = 0.15</th> -</tr> -</thead><tbody> -<tr> -<td style="text-align: center"><img src="img/quickstart-example/result003.png" alt="example1" style="width: 275px;"/></td> -<td style="text-align: center"><img src="img/quickstart-example/result008.png" alt="example2" style="width: 275px;"/></td> -<td style="text-align: center"><img src="img/quickstart-example/result015.png" alt="example3" style="width: 275px;"/></td> -</tr> -</tbody></table> +<div class="highlight"><pre><code class="language-bash"><span class="nb">cd </span>kmeans +python plotPoints.py result ./result clusters</code></pre></div> +<p>The following three pictures show the results for the sample input above. Play around with the parameters (number of iterations, number of clusters) to see how they affect the result.</p> + +<table> + <thead> + <tr> + <th style="text-align: center">relative stddev = 0.03</th> + <th style="text-align: center">relative stddev = 0.08</th> + <th style="text-align: center">relative stddev = 0.15</th> + </tr> + </thead> + <tbody> + <tr> + <td style="text-align: center"><img src="img/quickstart-example/result003.png" alt="example1" style="width: 275px;" /></td> + <td style="text-align: center"><img src="img/quickstart-example/result008.png" alt="example2" style="width: 275px;" /></td> + <td style="text-align: center"><img src="img/quickstart-example/result015.png" alt="example3" style="width: 275px;" /></td> + </tr> + </tbody> +</table> + + <!-- Disqus Area --> <div style="padding-top:30px" id="disqus_thread"></div> <script type="text/javascript"> @@ -256,14 +300,12 @@ cp /tmp/centers . })(); </script> <noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript";>comments powered by Disqus.</a></noscript> - <a href="http://disqus.com"; class="dsq-brlink">comments powered by <span class="logo-disqus">Disqus</span></a> - </div> </div> <div class="footer"> - <p><hr class="divider"></p> + <hr class="divider" /> <p><small>Apache Flink is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC. Incubation is @@ -274,9 +316,10 @@ incubation status is not necessarily a r stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.</small></p> -<p><a href="http://incubator.apache.org/";><img src="/img/apache-incubator-logo.png" alt="Incubator Logo"></a></p> +<p><a href="http://incubator.apache.org/";><img src="/img/apache-incubator-logo.png" alt="Incubator Logo" /></a></p> -<p class="text-center"><a href="/privacy-policy.html">Privacy Policy<a></p> +<p class="text-center"><a href="privacy-policy.html">Privacy Policy<a> +</a></a></p> </div> </div> Modified: flink/site/docs/0.6-incubating/scala_api_quickstart.html URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/scala_api_quickstart.html?rev=1657551&r1=1657550&r2=1657551&view=diff ============================================================================== --- flink/site/docs/0.6-incubating/scala_api_quickstart.html (original) +++ flink/site/docs/0.6-incubating/scala_api_quickstart.html Thu Feb 5 12:21:38 2015 @@ -5,116 +5,146 @@ <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1"> <title>Apache Flink (incubating): Quickstart: Scala API</title> - <link rel="stylesheet" href="/css/bootstrap.css"> - <link rel="stylesheet" href="/css/bootstrap-lumen-custom.css"> - <link rel="stylesheet" href="/css/syntax.css"> - <link rel="/css/custom.css"> - <link rel="css/codetabs.css"> - <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.1.0/css/font-awesome.min.css" rel="stylesheet"> + <link rel="shortcut icon" href="favicon.ico" type="image/x-icon"> + <link rel="icon" href="favicon.ico" type="image/x-icon"> + <link rel="stylesheet" href="css/bootstrap.css"> + <link rel="stylesheet" href="css/bootstrap-lumen-custom.css"> + <link rel="stylesheet" href="css/syntax.css"> + <link rel="stylesheet" href="css/custom.css"> + <link href="css/main/main.css" rel="stylesheet"> <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js";></script> - <script src="/js/bootstrap.min.js"></script> + <script src="js/bootstrap.min.js"></script> <script src="js/codetabs.js"></script> </head> <body> <nav class="navbar navbar-default navbar-fixed-top" role="navigation"> <div class="container"> - <div class="navbar-header"> - <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> - <span class="sr-only">Toggle navigation</span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <a class="navbar-brand" href="http://flink.incubator.apache.org";>Apache Flink</a> - </div> - - <div class="collapse navbar-collapse" id="navbar-collapse-1"> - <ul class="nav navbar-nav"> + <div class="row"> + <div class="col-md-1 af-mobile-nav-bar"> + <a href="index.html" title="Home"> + <img class="hidden-xs hidden-sm img-responsive" + src="img/logo.png" alt="Apache Flink Logo"> + </a> + <div class="row visible-xs"> + <div class="col-xs-3"> + <a href="index.html" title="Home"> + <img class="hidden-x hidden-sm img-responsive" + src="img/logo.png" alt="Apache Flink Logo"> + </a> + </div> + <div class="col-xs-5"></div> + <div class="col-xs-4"> + <div class="af-mobile-btn"> + <span class="glyphicon glyphicon-plus"></span> + </div> + </div> + </div> + </div> + <!-- Navigation --> + <div class="col-md-11"> + <div class="collapse navbar-collapse" id="navbar-collapse-1"> + <ul class="nav navbar-nav"> + + <li> + <a href="index.html" class="">Documentation</a> + </li> + + <li> + <a href="api/java/index.html">Javadoc</a> + </li> + + <li> + <a href="api/scala/index.html#org.apache.flink.api.scala.package">Scaladoc</a> + </li> - <li> - <a href="index.html" class="">Documentation</a> - </li> - - <li> - <a href="api/java/index.html">Javadoc</a> - </li> - </ul> + </ul> + </div> + </div> </div> </div> </nav> - <div style="padding-top:70px" class="container"> + + <div style="padding-top:120px" class="container"> <div class="row"> <div class="col-md-3"> <ul> - <li>Quickstart - <ul> - <li><a href="setup_quickstart.html">Install</a></li> - <li><a href="run_example_quickstart.html">Run Example</a></li> - <li><a href="java_api_quickstart.html">Java API</a></li> - <li><a href="scala_api_quickstart.html">Scala API</a></li> - <li><a href="faq.html">FAQ</a></li> - </ul> - </li> - - <li>Setup & Configuration - <ul> - <li><a href="building.html">Build Instructions</a></li> - <li><a href="local_setup.html">Local Setup</a></li> - <li><a href="cluster_setup.html">Cluster Setup</a></li> - <li><a href="yarn_setup.html">YARN Setup</a></li> - <li><a href="config.html">Configuration</a></li> - </ul> - </li> - - <li>Programming Guides - <ul> - <li><a href="java_api_guide.html">Java API</a></li> - <li><a href="java_api_transformations.html">Java API Transformations</a></li> - <li><a href="scala_api_guide.html">Scala API</a></li> - <li><a href="iterations.html">Iterations</a></li> - <li><a href="spargel_guide.html">Spargel Graph API</a></li> - </ul> - </li> - - <li>Examples - <ul> - <li><a href="java_api_examples.html">Java API</a></li> - <li><a href="scala_api_examples.html">Scala API</a></li> - <li><a href="example_connectors.html">Connecting to other systems</a></li> - </ul> - </li> - - <li>Execution - <ul> - <li><a href="local_execution.html">Local/Debugging</a></li> - <li><a href="cluster_execution.html">Cluster</a></li> - <li><a href="cli.html">Command-Line Interface</a></li> - <li><a href="web_client.html">Web Interface</a></li> - </ul> - </li> - - <li>Internals - <ul> - <li><a href="internal_overview.html">Overview</a></li> - <li><a href="internal_general_arch.html">General Architecture</a></li> - <li><a href="internal_add_operator.html">How-to: Adding a new Operator</a></li> - </ul> - </li> + <li><a href="faq.html">FAQ</a></li> + <li>Quickstart + <ul> + <li><a href="setup_quickstart.html">Setup</a></li> + <li><a href="run_example_quickstart.html">Run Example</a></li> + <li><a href="java_api_quickstart.html">Java API</a></li> + <li><a href="scala_api_quickstart.html">Scala API</a></li> + </ul> + </li> + + <li>Setup & Configuration + <ul> + <li><a href="local_setup.html">Local Setup</a></li> + <li><a href="building.html">Build Flink</a></li> + <li><a href="cluster_setup.html">Cluster Setup</a></li> + <li><a href="yarn_setup.html">YARN Setup</a></li> + <li><a href="config.html">Configuration</a></li> + </ul> + </li> + + <li>Programming Guides + <ul> + <li><a href="programming_guide.html">Programming Guide</a></li> + <li><a href="dataset_transformations.html">DataSet Transformations</a></li> + <li><a href="java8_programming_guide.html">Java 8 Programming Guide</a></li> + <li><a href="streaming_guide.html">Streaming Guide</a></li> + <li><a href="iterations.html">Iterations</a></li> + <li><a href="spargel_guide.html">Spargel Graph API</a></li> + <li><a href="hadoop_compatibility.html">Hadoop Compatibility</a></li> + </ul> + </li> + + <li>Examples + <ul> + <li><a href="examples.html">Bundled Examples</a></li> + <li><a href="example_connectors.html">Connecting to other systems</a></li> + </ul> + </li> + + <li>Execution + <ul> + <li><a href="local_execution.html">Local/Debugging</a></li> + <li><a href="cluster_execution.html">Cluster</a></li> + <li><a href="cli.html">Command-Line Interface</a></li> + <li><a href="web_client.html">Web Interface</a></li> + </ul> + </li> + + <li>Internals + <ul> + <li><a href="internal_overview.html">Overview</a></li> + </ul> + </li> </ul> + </div> <div class="col-md-9"> <h1>Quickstart: Scala API</h1> + + <ul id="markdown-toc"> + <li><a href="#requirements">Requirements</a></li> + <li><a href="#create-project">Create Project</a></li> + <li><a href="#inspect-project">Inspect Project</a></li> + <li><a href="#build-project">Build Project</a></li> + <li><a href="#next-steps">Next Steps</a></li> +</ul> - <p>Start working on your Flink Scala program in a few simple steps.</p> +<p>Start working on your Flink Scala program in a few simple steps.</p> -<h1 id="requirements">Requirements</h1> +<h2 id="requirements">Requirements</h2> <p>The only requirements are working <strong>Maven 3.0.4</strong> (or higher) and <strong>Java 6.x</strong> (or higher) installations.</p> -<h1 id="create-project">Create Project</h1> +<h2 id="create-project">Create Project</h2> <p>Use one of the following commands to <strong>create a project</strong>:</p> @@ -122,7 +152,6 @@ <li class="active"><a href="#quickstart-script" data-toggle="tab">Run the <strong>quickstart script</strong></a></li> <li><a href="#maven-archetype" data-toggle="tab">Use <strong>Maven archetypes</strong></a></li> </ul> - <div class="tab-content"> <div class="tab-pane active" id="quickstart-script"> @@ -134,49 +163,93 @@ <div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nv">$ </span>mvn archetype:generate <span class="se">\</span> -DarchetypeGroupId<span class="o">=</span>org.apache.flink <span class="se">\</span> -DarchetypeArtifactId<span class="o">=</span>flink-quickstart-scala <span class="se">\</span> - -DarchetypeVersion<span class="o">=</span>0.6.1-incubating</code></pre></div> + -DarchetypeVersion<span class="o">=</span>0.7.0-incubating</code></pre></div> This allows you to <strong>name your newly created project</strong>. It will interactively ask you for the groupId, artifactId, and package name. </div> </div> -<h1 id="inspect-project">Inspect Project</h1> +<h2 id="inspect-project">Inspect Project</h2> -<p>There will be a <strong>new directory in your working directory</strong>. If you've used the <em>curl</em> approach, the directory is called <code>quickstart</code>. Otherwise, it has the name of your artifactId.</p> +<p>There will be a new directory in your working directory. If youâve used the <em>curl</em> approach, the directory is called <code>quickstart</code>. Otherwise, it has the name of your artifactId.</p> -<p>The sample project is a <strong>Maven project</strong>, which contains a sample scala <em>job</em> that implements Word Count. Please note that the <em>RunJobLocal</em> and <em>RunJobRemote</em> objects allow you to start Flink in a development/testing mode.</p></p> +<p>The sample project is a <strong>Maven project</strong>, which contains two classes. <em>Job</em> is a basic skeleton program and <em>WordCountJob</em> a working example. Please note that the <em>main</em> method of both classes allow you to start Flink in a development/testing mode.</p> <p>We recommend to <strong>import this project into your IDE</strong>. For Eclipse, you need the following plugins, which you can install from the provided Eclipse Update Sites:</p> <ul> -<li><em>Eclipse 4.x</em> - -<ul> -<li><a href="http://download.scala-ide.org/sdk/e38/scala210/stable/site";>Scala IDE</a></li> -<li><a href="http://alchim31.free.fr/m2e-scala/update-site";>m2eclipse-scala</a></li> -<li><a href="https://repository.sonatype.org/content/repositories/forge-sites/m2e-extras/0.15.0/N/0.15.0.201206251206/";>Build Helper Maven Plugin</a></li> -</ul></li> -<li><em>Eclipse 3.7</em> - -<ul> -<li><a href="http://download.scala-ide.org/sdk/e37/scala210/stable/site";>Scala IDE</a></li> -<li><a href="http://alchim31.free.fr/m2e-scala/update-site";>m2eclipse-scala</a></li> -<li><a href="https://repository.sonatype.org/content/repositories/forge-sites/m2e-extras/0.14.0/N/0.14.0.201109282148/";>Build Helper Maven Plugin</a></li> -</ul></li> + <li><em>Eclipse 4.x</em> + <ul> + <li><a href="http://download.scala-ide.org/sdk/e38/scala210/stable/site";>Scala IDE</a></li> + <li><a href="http://alchim31.free.fr/m2e-scala/update-site";>m2eclipse-scala</a></li> + <li><a href="https://repository.sonatype.org/content/repositories/forge-sites/m2e-extras/0.15.0/N/0.15.0.201206251206/";>Build Helper Maven Plugin</a></li> + </ul> + </li> + <li><em>Eclipse 3.7</em> + <ul> + <li><a href="http://download.scala-ide.org/sdk/e37/scala210/stable/site";>Scala IDE</a></li> + <li><a href="http://alchim31.free.fr/m2e-scala/update-site";>m2eclipse-scala</a></li> + <li><a href="https://repository.sonatype.org/content/repositories/forge-sites/m2e-extras/0.14.0/N/0.14.0.201109282148/";>Build Helper Maven Plugin</a></li> + </ul> + </li> </ul> <p>The IntelliJ IDE also supports Maven and offers a plugin for Scala development.</p> -<h1 id="build-project">Build Project</h1> +<h2 id="build-project">Build Project</h2> + +<p>If you want to <strong>build your project</strong>, go to your project directory and issue the<code>mvn clean package</code> command. You will <strong>find a jar</strong> that runs on every Flink cluster in <strong>target/your-artifact-id-1.0-SNAPSHOT.jar</strong>. There is also a fat-jar, <strong>target/your-artifact-id-1.0-SNAPSHOT-flink-fat-jar.jar</strong>. This +also contains all dependencies that get added to the maven project.</p> + +<h2 id="next-steps">Next Steps</h2> -<p>If you want to <strong>build your project</strong>, go to your project directory and issue the<code>mvn clean package</code> command. You will <strong>find a jar</strong> that runs on every Flink cluster in <strong>target/flink-project-0.1-SNAPSHOT.jar</strong>.</p> +<p>Write your application!</p> -<h1 id="next-steps">Next Steps</h1> +<p>The quickstart project contains a WordCount implementation, the âHello Worldâ of Big Data processing systems. The goal of WordCount is to determine the frequencies of words in a text, e.g., how often do the terms âtheâ or âhouseâ occurs in all Wikipedia texts.</p> -<p><strong>Write your application!</strong> -If you have any trouble, ask on our <a href="https://issues.apache.org/jira/browse/FLINK";>Jira page</a> (open an issue) or on our Mailing list. We are happy to provide help.</p> +<p><strong>Sample Input</strong>:</p> +<div class="highlight"><pre><code class="language-bash">big data is big</code></pre></div> +<p><strong>Sample Output</strong>:</p> + +<div class="highlight"><pre><code class="language-bash">big 2 +data 1 +is 1</code></pre></div> + +<p>The following code shows the WordCount implementation from the Quickstart which processes some text lines with two operators (FlatMap and Reduce), and writes the prints the resulting words and counts to std-out.</p> + +<div class="highlight"><pre><code class="language-scala"><span class="k">object</span> <span class="nc">WordCountJob</span> <span class="o">{</span> + <span class="k">def</span> <span class="n">main</span><span class="o">(</span><span class="n">args</span><span class="k">:</span> <span class="kt">Array</span><span class="o">[</span><span class="kt">String</span><span class="o">])</span> <span class="o">{</span> + + <span class="c1">// set up the execution environment</span> + <span class="k">val</span> <span class="n">env</span> <span class="k">=</span> <span class="nc">ExecutionEnvironment</span><span class="o">.</span><span class="n">getExecutionEnvironment</span> + + <span class="c1">// get input data</span> + <span class="k">val</span> <span class="n">text</span> <span class="k">=</span> <span class="n">env</span><span class="o">.</span><span class="n">fromElements</span><span class="o">(</span><span class="s">"To be, or not to be,--that is the question:--"</span><span class="o">,</span> + <span class="s">"Whether 'tis nobler in the mind to suffer"</span><span class="o">,</span> <span class="s">"The slings and arrows of outrageous fortune"</span><span class="o">,</span> + <span class="s">"Or to take arms against a sea of troubles,"</span><span class="o">)</span> + + <span class="k">val</span> <span class="n">counts</span> <span class="k">=</span> <span class="n">text</span><span class="o">.</span><span class="n">flatMap</span> <span class="o">{</span> <span class="k">_</span><span class="o">.</span><span class="n">toLowerCase</span><span class="o">.</span><span class="n">split</span><span class="o">(</span><span class="s">"\\W+"</span><span class="o">)</span> <span class="o">}</span> + <span class="o">.</span><span class="n">map</span> <span class="o">{</span> <span class="o">(</span><span class="k">_</span><span class="o">,</span> <span class="mi">1</span><span class="o">)</span> <span class="o">}</span> + <span class="o">.</span><span class="n">groupBy</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> + <span class="o">.</span><span class="n">sum</span><span class="o">(</span><span class="mi">1</span><span class="o">)</span> + + <span class="c1">// emit result</span> + <span class="n">counts</span><span class="o">.</span><span class="n">print</span><span class="o">()</span> + + <span class="c1">// execute program</span> + <span class="n">env</span><span class="o">.</span><span class="n">execute</span><span class="o">(</span><span class="s">"WordCount Example"</span><span class="o">)</span> + <span class="o">}</span> +<span class="o">}</span></code></pre></div> + +<p><a href="https://github.com/apache/incubator-flink/blob/master//flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/wordcount/WordCount.scala";>Check GitHub</a> for the full example code.</p> + +<p>For a complete overview over our API, have a look at the <a href="programming_guide.html">Programming Guide</a> and <a href="examples.html">further example programs</a>. If you have any trouble, ask on our <a href="http://mail-archives.apache.org/mod_mbox/incubator-flink-dev/";>Mailing List</a>. We are happy to provide help.</p> + + + + <!-- Disqus Area --> <div style="padding-top:30px" id="disqus_thread"></div> <script type="text/javascript"> @@ -191,14 +264,12 @@ If you have any trouble, ask on our <a h })(); </script> <noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript";>comments powered by Disqus.</a></noscript> - <a href="http://disqus.com"; class="dsq-brlink">comments powered by <span class="logo-disqus">Disqus</span></a> - </div> </div> <div class="footer"> - <p><hr class="divider"></p> + <hr class="divider" /> <p><small>Apache Flink is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC. Incubation is @@ -209,9 +280,10 @@ incubation status is not necessarily a r stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.</small></p> -<p><a href="http://incubator.apache.org/";><img src="/img/apache-incubator-logo.png" alt="Incubator Logo"></a></p> +<p><a href="http://incubator.apache.org/";><img src="/img/apache-incubator-logo.png" alt="Incubator Logo" /></a></p> -<p class="text-center"><a href="/privacy-policy.html">Privacy Policy<a></p> +<p class="text-center"><a href="privacy-policy.html">Privacy Policy<a> +</a></a></p> </div> </div> Modified: flink/site/docs/0.6-incubating/setup_quickstart.html URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/setup_quickstart.html?rev=1657551&r1=1657550&r2=1657551&view=diff ============================================================================== --- flink/site/docs/0.6-incubating/setup_quickstart.html (original) +++ flink/site/docs/0.6-incubating/setup_quickstart.html Thu Feb 5 12:21:38 2015 @@ -5,219 +5,259 @@ <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1"> <title>Apache Flink (incubating): Quickstart: Setup</title> - <link rel="stylesheet" href="/css/bootstrap.css"> - <link rel="stylesheet" href="/css/bootstrap-lumen-custom.css"> - <link rel="stylesheet" href="/css/syntax.css"> - <link rel="/css/custom.css"> - <link rel="css/codetabs.css"> - <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.1.0/css/font-awesome.min.css" rel="stylesheet"> + <link rel="shortcut icon" href="favicon.ico" type="image/x-icon"> + <link rel="icon" href="favicon.ico" type="image/x-icon"> + <link rel="stylesheet" href="css/bootstrap.css"> + <link rel="stylesheet" href="css/bootstrap-lumen-custom.css"> + <link rel="stylesheet" href="css/syntax.css"> + <link rel="stylesheet" href="css/custom.css"> + <link href="css/main/main.css" rel="stylesheet"> <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js";></script> - <script src="/js/bootstrap.min.js"></script> + <script src="js/bootstrap.min.js"></script> <script src="js/codetabs.js"></script> </head> <body> <nav class="navbar navbar-default navbar-fixed-top" role="navigation"> <div class="container"> - <div class="navbar-header"> - <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> - <span class="sr-only">Toggle navigation</span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <a class="navbar-brand" href="http://flink.incubator.apache.org";>Apache Flink</a> - </div> - - <div class="collapse navbar-collapse" id="navbar-collapse-1"> - <ul class="nav navbar-nav"> + <div class="row"> + <div class="col-md-1 af-mobile-nav-bar"> + <a href="index.html" title="Home"> + <img class="hidden-xs hidden-sm img-responsive" + src="img/logo.png" alt="Apache Flink Logo"> + </a> + <div class="row visible-xs"> + <div class="col-xs-3"> + <a href="index.html" title="Home"> + <img class="hidden-x hidden-sm img-responsive" + src="img/logo.png" alt="Apache Flink Logo"> + </a> + </div> + <div class="col-xs-5"></div> + <div class="col-xs-4"> + <div class="af-mobile-btn"> + <span class="glyphicon glyphicon-plus"></span> + </div> + </div> + </div> + </div> + <!-- Navigation --> + <div class="col-md-11"> + <div class="collapse navbar-collapse" id="navbar-collapse-1"> + <ul class="nav navbar-nav"> + + <li> + <a href="index.html" class="">Documentation</a> + </li> + + <li> + <a href="api/java/index.html">Javadoc</a> + </li> + + <li> + <a href="api/scala/index.html#org.apache.flink.api.scala.package">Scaladoc</a> + </li> - <li> - <a href="index.html" class="">Documentation</a> - </li> - - <li> - <a href="api/java/index.html">Javadoc</a> - </li> - </ul> + </ul> + </div> + </div> </div> </div> </nav> - <div style="padding-top:70px" class="container"> + + <div style="padding-top:120px" class="container"> <div class="row"> <div class="col-md-3"> <ul> - <li>Quickstart - <ul> - <li><a href="setup_quickstart.html">Install</a></li> - <li><a href="run_example_quickstart.html">Run Example</a></li> - <li><a href="java_api_quickstart.html">Java API</a></li> - <li><a href="scala_api_quickstart.html">Scala API</a></li> - <li><a href="faq.html">FAQ</a></li> - </ul> - </li> - - <li>Setup & Configuration - <ul> - <li><a href="building.html">Build Instructions</a></li> - <li><a href="local_setup.html">Local Setup</a></li> - <li><a href="cluster_setup.html">Cluster Setup</a></li> - <li><a href="yarn_setup.html">YARN Setup</a></li> - <li><a href="config.html">Configuration</a></li> - </ul> - </li> - - <li>Programming Guides - <ul> - <li><a href="java_api_guide.html">Java API</a></li> - <li><a href="java_api_transformations.html">Java API Transformations</a></li> - <li><a href="scala_api_guide.html">Scala API</a></li> - <li><a href="iterations.html">Iterations</a></li> - <li><a href="spargel_guide.html">Spargel Graph API</a></li> - </ul> - </li> - - <li>Examples - <ul> - <li><a href="java_api_examples.html">Java API</a></li> - <li><a href="scala_api_examples.html">Scala API</a></li> - <li><a href="example_connectors.html">Connecting to other systems</a></li> - </ul> - </li> - - <li>Execution - <ul> - <li><a href="local_execution.html">Local/Debugging</a></li> - <li><a href="cluster_execution.html">Cluster</a></li> - <li><a href="cli.html">Command-Line Interface</a></li> - <li><a href="web_client.html">Web Interface</a></li> - </ul> - </li> - - <li>Internals - <ul> - <li><a href="internal_overview.html">Overview</a></li> - <li><a href="internal_general_arch.html">General Architecture</a></li> - <li><a href="internal_add_operator.html">How-to: Adding a new Operator</a></li> - </ul> - </li> + <li><a href="faq.html">FAQ</a></li> + <li>Quickstart + <ul> + <li><a href="setup_quickstart.html">Setup</a></li> + <li><a href="run_example_quickstart.html">Run Example</a></li> + <li><a href="java_api_quickstart.html">Java API</a></li> + <li><a href="scala_api_quickstart.html">Scala API</a></li> + </ul> + </li> + + <li>Setup & Configuration + <ul> + <li><a href="local_setup.html">Local Setup</a></li> + <li><a href="building.html">Build Flink</a></li> + <li><a href="cluster_setup.html">Cluster Setup</a></li> + <li><a href="yarn_setup.html">YARN Setup</a></li> + <li><a href="config.html">Configuration</a></li> + </ul> + </li> + + <li>Programming Guides + <ul> + <li><a href="programming_guide.html">Programming Guide</a></li> + <li><a href="dataset_transformations.html">DataSet Transformations</a></li> + <li><a href="java8_programming_guide.html">Java 8 Programming Guide</a></li> + <li><a href="streaming_guide.html">Streaming Guide</a></li> + <li><a href="iterations.html">Iterations</a></li> + <li><a href="spargel_guide.html">Spargel Graph API</a></li> + <li><a href="hadoop_compatibility.html">Hadoop Compatibility</a></li> + </ul> + </li> + + <li>Examples + <ul> + <li><a href="examples.html">Bundled Examples</a></li> + <li><a href="example_connectors.html">Connecting to other systems</a></li> + </ul> + </li> + + <li>Execution + <ul> + <li><a href="local_execution.html">Local/Debugging</a></li> + <li><a href="cluster_execution.html">Cluster</a></li> + <li><a href="cli.html">Command-Line Interface</a></li> + <li><a href="web_client.html">Web Interface</a></li> + </ul> + </li> + + <li>Internals + <ul> + <li><a href="internal_overview.html">Overview</a></li> + </ul> + </li> </ul> + </div> <div class="col-md-9"> <h1>Quickstart: Setup</h1> + + <ul id="markdown-toc"> + <li><a href="#requirements">Requirements</a></li> + <li><a href="#download">Download</a></li> + <li><a href="#start">Start</a></li> + <li><a href="#run-example">Run Example</a></li> + <li><a href="#cluster-setup">Cluster Setup</a></li> + <li><a href="#flink-on-yarn">Flink on YARN</a></li> +</ul> - <p>Get Flink up and running in a few simple steps.</p> - -<h1 id="requirements">Requirements</h1> +<p>Get Flink up and running in a few simple steps.</p> +<h2 id="requirements">Requirements</h2> <p>Flink runs on all <strong>UNIX-like</strong> environments: <strong>Linux</strong>, <strong>Mac OS X</strong>, <strong>Cygwin</strong>. The only requirement is to have a working <strong>Java 6.x</strong> (or higher) installation.</p> -<h1 id="download">Download</h1> - +<h2 id="download">Download</h2> <p>Download the ready to run binary package. Choose the Flink distribution that <strong>matches your Hadoop version</strong>. If you are unsure which version to choose or you just want to run locally, pick the package for Hadoop 1.2.</p> <ul class="nav nav-tabs"> - <li class="active"><a href="#bin-hadoop1" data-toggle="tab">Hadoop 1.2</a></li> - <li><a href="#bin-hadoop2" data-toggle="tab">Hadoop 2 (YARN)</a></li> - </ul> - -<p><div class="tab-content text-center"> - <div class="tab-pane active" id="bin-hadoop1"> - <a class="btn btn-info btn-lg" onclick="_gaq.push(['_trackEvent','Action','download-quickstart-setup-1',this.href]);" href="http://www.apache.org/dyn/closer.cgi/incubator/flink/flink-0.6.1-incubating/flink-0.6.1-incubating-bin-hadoop1.tgz";><i class="icon-download"> </i> Download Flink for Hadoop 1.2</a> - </div> - <div class="tab-pane" id="bin-hadoop2"> - <a class="btn btn-info btn-lg" onclick="_gaq.push(['_trackEvent','Action','download-quickstart-setup-2',this.href]);" href="http://www.apache.org/dyn/closer.cgi/incubator/flink/flink-0.6.1-incubating/flink-0.6.1-incubating-bin-hadoop2.tgz";><i class="icon-download"> </i> Download Flink for Hadoop 2</a> - </div> - </div> -</p></p> - -<h1 id="start">Start</h1> + <li class="active"><a href="#bin-hadoop1" data-toggle="tab">Hadoop 1.2</a></li> + <li><a href="#bin-hadoop2" data-toggle="tab">Hadoop 2 (YARN)</a></li> +</ul> +<p> +<div class="tab-content text-center"> + <div class="tab-pane active" id="bin-hadoop1"> + <a class="btn btn-info btn-lg" onclick="_gaq.push(['_trackEvent','Action','download-quickstart-setup-1',this.href]);" href="http://www.apache.org/dyn/closer.cgi/incubator/flink/flink-0.7.0-incubating/flink-0.7.0-incubating-bin-hadoop1.tgz";><i class="icon-download"> </i> Download Flink for Hadoop 1.2</a> + </div> + <div class="tab-pane" id="bin-hadoop2"> + <a class="btn btn-info btn-lg" onclick="_gaq.push(['_trackEvent','Action','download-quickstart-setup-2',this.href]);" href="http://www.apache.org/dyn/closer.cgi/incubator/flink/flink-0.7.0-incubating/flink-0.7.0-incubating-bin-hadoop2.tgz";><i class="icon-download"> </i> Download Flink for Hadoop 2</a> + </div> +</div> +</p> +<h2 id="start">Start</h2> <p>You are almost done.</p> <ol> -<li>Go to the download directory.</li> -<li>Unpack the downloaded archive.</li> -<li>Start Flink.</li> + <li>Go to the download directory.</li> + <li>Unpack the downloaded archive.</li> + <li>Start Flink.</li> </ol> -<div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nv">$ </span><span class="nb">cd</span> ~/Downloads <span class="c"># Go to download directory</span> + +<div class="highlight"><pre><code class="language-bash"><span class="nv">$ </span><span class="nb">cd</span> ~/Downloads <span class="c"># Go to download directory</span> <span class="nv">$ </span>tar xzf flink-*.tgz <span class="c"># Unpack the downloaded archive</span> <span class="nv">$ </span><span class="nb">cd </span>flink -<span class="nv">$ </span>bin/start-local.sh <span class="c"># Start Flink</span> -</code></pre></div> -<p>Check the <strong>JobManager's web frontend</strong> at <a href="http://localhost:8081";>http://localhost:8081</a> and make sure everything is up and running.</p> +<span class="nv">$ </span>bin/start-local.sh <span class="c"># Start Flink</span></code></pre></div> -<h1 id="run-example">Run Example</h1> +<p>Check the <strong>JobManagerâs web frontend</strong> at <a href="http://localhost:8081";>http://localhost:8081</a> and make +sure everything is up and running.</p> + +<h2 id="run-example">Run Example</h2> <p>Run the <strong>Word Count example</strong> to see Flink at work.</p> <ul> -<li><strong>Download test data</strong>: -<code>bash -$ wget -O hamlet.txt http://www.gutenberg.org/cache/epub/1787/pg1787.txt -</code></li> -<li>You now have a text file called <em>hamlet.txt</em> in your working directory.</li> -<li><p><strong>Start the example program</strong>:</p> -<div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nv">$ </span>bin/flink run <span class="se">\</span> ---jarfile ./examples/flink-java-examples-0.6.1-incubating-WordCount.jar <span class="se">\</span> - ---arguments file://<span class="sb">`</span><span class="nb">pwd</span><span class="sb">`</span>/hamlet.txt file://<span class="sb">`</span><span class="nb">pwd</span><span class="sb">`</span>/wordcount-result.txt -</code></pre></div></li> -<li><p>You will find a file called <strong>wordcount-result.txt</strong> in your current directory.</p></li> + <li> + <p><strong>Download test data</strong>:</p> + + <div class="highlight"><pre><code class="language-bash"><span class="nv">$ </span>wget -O hamlet.txt http://www.gutenberg.org/cache/epub/1787/pg1787.txt</code></pre></div> + </li> + <li>You now have a text file called <em>hamlet.txt</em> in your working directory.</li> + <li> + <p><strong>Start the example program</strong>:</p> + + <div class="highlight"><pre><code class="language-bash"><span class="nv">$ </span>bin/flink run <span class="se">\</span> + --jarfile ./examples/flink-java-examples-0.7.0-incubating-WordCount.jar <span class="se">\</span> + + --arguments file://<span class="sb">`</span><span class="nb">pwd</span><span class="sb">`</span>/hamlet.txt file://<span class="sb">`</span><span class="nb">pwd</span><span class="sb">`</span>/wordcount-result.txt</code></pre></div> + </li> + <li>You will find a file called <strong>wordcount-result.txt</strong> in your current directory.</li> </ul> -<h1 id="cluster-setup">Cluster Setup</h1> +<h2 id="cluster-setup">Cluster Setup</h2> -<p><strong>Running Flink on a cluster</strong> is as easy as running it locally. Having <strong>passwordless SSH</strong> and <strong>the same directory structure</strong> on all your cluster nodes lets you use our scripts to control everything.</p> +<p><strong>Running Flink on a cluster</strong> is as easy as running it locally. Having <strong>passwordless SSH</strong> and +<strong>the same directory structure</strong> on all your cluster nodes lets you use our scripts to control +everything.</p> <ol> -<li>Copy the unpacked <strong>flink</strong> directory from the downloaded archive to the same file system path on each node of your setup.</li> -<li>Choose a <strong>master node</strong> (JobManager) and set the <code>jobmanager.rpc.address</code> key in <code>conf/flink-conf.yaml</code> to its IP or hostname. Make sure that all nodes in your cluster have the same <code>jobmanager.rpc.address</code> configured.</li> -<li>Add the IPs or hostnames (one per line) of all <strong>worker nodes</strong> (TaskManager) to the slaves files in <code>conf/slaves</code>.</li> + <li>Copy the unpacked <strong>flink</strong> directory from the downloaded archive to the same file system path +on each node of your setup.</li> + <li>Choose a <strong>master node</strong> (JobManager) and set the <code>jobmanager.rpc.address</code> key in +<code>conf/flink-conf.yaml</code> to its IP or hostname. Make sure that all nodes in your cluster have the same +<code>jobmanager.rpc.address</code> configured.</li> + <li>Add the IPs or hostnames (one per line) of all <strong>worker nodes</strong> (TaskManager) to the slaves files +in <code>conf/slaves</code>.</li> </ol> <p>You can now <strong>start the cluster</strong> at your master node with <code>bin/start-cluster.sh</code>.</p> -<p>The following <strong>example</strong> illustrates the setup with three nodes (with IP addresses from <em>10.0.0.1</em> to <em>10.0.0.3</em> and hostnames <em>master</em>, <em>worker1</em>, <em>worker2</em>) and shows the contents of the configuration files, which need to be accessible at the same path on all machines:</p> +<p>The following <strong>example</strong> illustrates the setup with three nodes (with IP addresses from <em>10.0.0.1</em> +to <em>10.0.0.3</em> and hostnames <em>master</em>, <em>worker1</em>, <em>worker2</em>) and shows the contents of the +configuration files, which need to be accessible at the same path on all machines:</p> <div class="row"> <div class="col-md-6 text-center"> - <img src="img/quickstart_cluster.png" style="width: 85%"> + <img src="img/quickstart_cluster.png" style="width: 85%" /> </div> <div class="col-md-6"> <div class="row"> <p class="lead text-center"> - /path/to/<strong>flink/conf/<br>flink-conf.yaml</strong> + /path/to/<strong>flink/conf/<br />flink-conf.yaml</strong> <pre>jobmanager.rpc.address: 10.0.0.1</pre> </p> </div> <div class="row" style="margin-top: 1em;"> <p class="lead text-center"> - /path/to/<strong>flink/<br>conf/slaves</strong> + /path/to/<strong>flink/<br />conf/slaves</strong> <pre> 10.0.0.2 10.0.0.3</pre> </p> </div> +</div> +</div> -<p></div> -</div></p> - -<h1 id="flink-on-yarn">Flink on YARN</h1> - +<h2 id="flink-on-yarn">Flink on YARN</h2> <p>You can easily deploy Flink on your existing <strong>YARN cluster</strong>. </p> <ol> -<li>Download the <strong>Flink YARN package</strong> with the YARN client: <a href="http://www.apache.org/dyn/closer.cgi/incubator/flink/flink-0.6.1-incubating/flink-0.6.1-incubating-bin-hadoop2-yarn.tgz";>Flink for YARN</a></li> -<li>Make sure your <strong>HADOOP_HOME</strong> (or <em>YARN_CONF_DIR</em> or <em>HADOOP_CONF_DIR</em>) <strong>environment variable</strong> is set to read your YARN and HDFS configuration.</li> -<li>Run the <strong>YARN client</strong> with: <code>./bin/yarn-session.sh</code>. You can run the client with options <code>-n 10 -tm 8192</code> to allocate 10 TaskManagers with 8GB of memory each.</li> + <li>Download the <strong>Flink YARN package</strong> with the YARN client: <a href="http://www.apache.org/dyn/closer.cgi/incubator/flink/flink-0.7.0-incubating/flink-0.7.0-incubating-bin-hadoop2-yarn.tgz";>Flink for YARN</a></li> + <li>Make sure your <strong>HADOOP_HOME</strong> (or <em>YARN_CONF_DIR</em> or <em>HADOOP_CONF_DIR</em>) <strong>environment variable</strong> is set to read your YARN and HDFS configuration.</li> + <li>Run the <strong>YARN client</strong> with: <code>./bin/yarn-session.sh</code>. You can run the client with options <code>-n 10 -tm 8192</code> to allocate 10 TaskManagers with 8GB of memory each.</li> </ol> <p>For <strong>more detailed instructions</strong>, check out the programming Guides and examples.</p> - + + <!-- Disqus Area --> <div style="padding-top:30px" id="disqus_thread"></div> <script type="text/javascript"> @@ -232,14 +272,12 @@ $ wget -O hamlet.txt http://www.gutenber })(); </script> <noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript";>comments powered by Disqus.</a></noscript> - <a href="http://disqus.com"; class="dsq-brlink">comments powered by <span class="logo-disqus">Disqus</span></a> - </div> </div> <div class="footer"> - <p><hr class="divider"></p> + <hr class="divider" /> <p><small>Apache Flink is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC. Incubation is @@ -250,9 +288,10 @@ incubation status is not necessarily a r stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.</small></p> -<p><a href="http://incubator.apache.org/";><img src="/img/apache-incubator-logo.png" alt="Incubator Logo"></a></p> +<p><a href="http://incubator.apache.org/";><img src="/img/apache-incubator-logo.png" alt="Incubator Logo" /></a></p> -<p class="text-center"><a href="/privacy-policy.html">Privacy Policy<a></p> +<p class="text-center"><a href="privacy-policy.html">Privacy Policy<a> +</a></a></p> </div> </div>
