http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/0.9/setup/yarn_setup.html ---------------------------------------------------------------------- diff --git a/content/docs/0.9/setup/yarn_setup.html b/content/docs/0.9/setup/yarn_setup.html deleted file mode 100644 index e45d2c2..0000000 --- a/content/docs/0.9/setup/yarn_setup.html +++ /dev/null @@ -1,443 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<!DOCTYPE html> - -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - - <title>Apache Flink 0.9.0 Documentation: YARN Setup</title> - - <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css";> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - - - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg";></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li><a href="http://flink.apache.org/docs/0.9/index.html";>Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li> - - <!-- Setup --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/setup"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/setup/building.html";>Get Flink 0.9-SNAPSHOT</a></li> - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html"; class="active">Local</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html";>Cluster (Standalone)</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html";>YARN</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html";>GCloud</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html";>Flink on Tez <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/config.html";>Configuration</a></li> - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/apis"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html";><strong>Batch: DataSet API</strong></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html";><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/python.html";>Python API <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="scala_shell.html">Interactive Scala Shell</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html";>Dataset Transformations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html";>Best Practices</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html";>Connectors</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html";>Examples</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html";>Local Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html";>Cluster Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html";>Command Line Interface</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html";>Web Client</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html";>Iterations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html";>Java 8</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html";>Hadoop Compatability <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/libs"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html";>Graphs: Spargel</a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html";>Graphs: Gelly <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/ml/";>Machine Learning <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/table.html";>Relational: Table <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Internals --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/internals"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html";>How to Contribute</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html";>Coding Guidelines</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html";>IDE Setup</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html";>Logging</a></li> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Internals</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html";>Architecture & Process Model</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html";>Type Extraction & Serialization</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html";>Jobs & Scheduling</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html";>How-To: Add an Operator</a></li> - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html";> - <div class="form-group"> - <input type="text" class="form-control" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-10 col-sm-offset-1"> - <h1>YARN Setup</h1> - - - -<ul id="markdown-toc"> - <li><a href="#quickstart-start-a-long-running-flink-cluster-on-yarn" id="markdown-toc-quickstart-start-a-long-running-flink-cluster-on-yarn">Quickstart: Start a long-running Flink cluster on YARN</a></li> - <li><a href="#quickstart-run-a-flink-job-on-yarn" id="markdown-toc-quickstart-run-a-flink-job-on-yarn">Quickstart: Run a Flink job on YARN</a></li> - <li><a href="#apache-flink-on-hadoop-yarn-using-a-yarn-session" id="markdown-toc-apache-flink-on-hadoop-yarn-using-a-yarn-session">Apache Flink on Hadoop YARN using a YARN Session</a> <ul> - <li><a href="#start-flink-session" id="markdown-toc-start-flink-session">Start Flink Session</a></li> - <li><a href="#submit-job-to-flink" id="markdown-toc-submit-job-to-flink">Submit Job to Flink</a></li> - </ul> - </li> - <li><a href="#run-a-single-flink-job-on-hadoop-yarn" id="markdown-toc-run-a-single-flink-job-on-hadoop-yarn">Run a single Flink job on Hadoop YARN</a></li> - <li><a href="#recovery-behavior-of-flink-on-yarn" id="markdown-toc-recovery-behavior-of-flink-on-yarn">Recovery behavior of Flink on YARN</a></li> - <li><a href="#debugging-a-failed-yarn-session" id="markdown-toc-debugging-a-failed-yarn-session">Debugging a failed YARN session</a> <ul> - <li><a href="#log-files" id="markdown-toc-log-files">Log Files</a></li> - <li><a href="#yarn-client-console--webinterfaces" id="markdown-toc-yarn-client-console--webinterfaces">YARN Client console & Webinterfaces</a></li> - </ul> - </li> - <li><a href="#build-yarn-client-for-a-specific-hadoop-version" id="markdown-toc-build-yarn-client-for-a-specific-hadoop-version">Build YARN client for a specific Hadoop version</a></li> - <li><a href="#background--internals" id="markdown-toc-background--internals">Background / Internals</a></li> -</ul> - -<h2 id="quickstart-start-a-long-running-flink-cluster-on-yarn">Quickstart: Start a long-running Flink cluster on YARN</h2> - -<p>Start a YARN session with 4 Task Managers (each with 4 GB of Heapspace):</p> - -<div class="highlight"><pre><code class="language-bash"><span class="c"># get the hadoop2 package from the Flink download page at</span> -<span class="c"># http://flink.apache.org/downloads.html</span> -tar xvzf flink-0.9.0-bin-hadoop2.tgz -<span class="nb">cd </span>flink-0.9.0/ -./bin/yarn-session.sh -n <span class="m">4</span> -jm <span class="m">1024</span> -tm 4096</code></pre></div> - -<p>Specify the <code>-s</code> flag for the number of processing slots per Task Manager. We recommend to set the number of slots to the number of processors per machine.</p> - -<p>Once the session has been started, you can submit jobs to the cluster using the <code>./bin/flink</code> tool.</p> - -<h2 id="quickstart-run-a-flink-job-on-yarn">Quickstart: Run a Flink job on YARN</h2> - -<div class="highlight"><pre><code class="language-bash"><span class="c"># get the hadoop2 package from the Flink download page at</span> -<span class="c"># http://flink.apache.org/downloads.html</span> -tar xvzf flink-0.9.0-bin-hadoop2.tgz -<span class="nb">cd </span>flink-0.9.0/ -./bin/flink run -m yarn-cluster -yn <span class="m">4</span> -yjm <span class="m">1024</span> -ytm <span class="m">4096</span> ./examples/flink-java-examples-0.9.0-WordCount.jar</code></pre></div> - -<h2 id="apache-flink-on-hadoop-yarn-using-a-yarn-session">Apache Flink on Hadoop YARN using a YARN Session</h2> - -<p>Apache <a href="http://hadoop.apache.org/";>Hadoop YARN</a> is a cluster resource management framework. It allows to run various distributed applications on top of a cluster. Flink runs on YARN next to other applications. Users do not have to setup or install anything if there is already a YARN setup.</p> - -<p><strong>Requirements</strong></p> - -<ul> - <li>Apache Hadoop 2.2</li> - <li>HDFS (Hadoop Distributed File System) (or another distributed file system supported by Hadoop)</li> -</ul> - -<p>If you have troubles using the Flink YARN client, have a look in the <a href="http://flink.apache.org/docs/0.9/faq.html";>FAQ section</a>.</p> - -<h3 id="start-flink-session">Start Flink Session</h3> - -<p>Follow these instructions to learn how to launch a Flink Session within your YARN cluster.</p> - -<p>A session will start all required Flink services (JobManager and TaskManagers) so that you can submit programs to the cluster. Note that you can run multiple programs per session.</p> - -<h4 id="download-flink-for-yarn">Download Flink for YARN</h4> - -<p>Download the YARN tgz package on the <a href="http://flink.apache.org/docs/0.9/downloads.html";>download page</a>. It contains the required files.</p> - -<p>Extract the package using:</p> - -<div class="highlight"><pre><code class="language-bash">tar xvzf flink-0.9.0-bin-hadoop2.tgz -<span class="nb">cd </span>flink-0.9.0/</code></pre></div> - -<p>If you want to build the YARN .tgz file from sources, follow the <a href="building.html">build instructions</a>. You can find the result of the build in <code>flink-dist/target/flink-0.9.0-bin/flink-0.9.0/</code> (<em>Note: The version might be different for you</em> ).</p> - -<h4 id="start-a-session">Start a Session</h4> - -<p>Use the following command to start a session</p> - -<div class="highlight"><pre><code class="language-bash">./bin/yarn-session.sh</code></pre></div> - -<p>This command will show you the following overview:</p> - -<div class="highlight"><pre><code class="language-bash">Usage: - Required - -n,--container <arg> Number of YARN container to allocate <span class="o">(=</span>Number of Task Managers<span class="o">)</span> - Optional - -D <arg> Dynamic properties - -d,--detached Start detached - -jm,--jobManagerMemory <arg> Memory <span class="k">for</span> JobManager Container <span class="o">[</span>in MB<span class="o">]</span> - -q,--query Display available YARN resources <span class="o">(</span>memory, cores<span class="o">)</span> - -qu,--queue <arg> Specify YARN queue. - -s,--slots <arg> Number of slots per TaskManager - -st,--streaming Start Flink in streaming mode - -tm,--taskManagerMemory <arg> Memory per TaskManager Container <span class="o">[</span>in MB<span class="o">]</span></code></pre></div> - -<p>Please note that the Client requires the <code>YARN_CONF_DIR</code> or <code>HADOOP_CONF_DIR</code> environment variable to be set to read the YARN and HDFS configuration.</p> - -<p><strong>Example:</strong> Issue the following command to allocate 10 Task Managers, with 8 GB of memory and 32 processing slots each:</p> - -<div class="highlight"><pre><code class="language-bash">./bin/yarn-session.sh -n <span class="m">10</span> -tm <span class="m">8192</span> -s 32</code></pre></div> - -<p>The system will use the configuration in <code>conf/flink-config.yaml</code>. Please follow our <a href="config.html">configuration guide</a> if you want to change something.</p> - -<p>Flink on YARN will overwrite the following configuration parameters <code>jobmanager.rpc.address</code> (because the JobManager is always allocated at different machines), <code>taskmanager.tmp.dirs</code> (we are using the tmp directories given by YARN) and <code>parallelism.default</code> if the number of slots has been specified.</p> - -<p>If you donât want to change the configuration file to set configuration parameters, there is the option to pass dynamic properties via the <code>-D</code> flag. So you can pass parameters this way: <code>-Dfs.overwrite-files=true -Dtaskmanager.network.numberOfBuffers=16368</code>.</p> - -<p>The example invocation starts 11 containers, since there is one additional container for the ApplicationMaster and Job Manager.</p> - -<p>Once Flink is deployed in your YARN cluster, it will show you the connection details of the Job Manager.</p> - -<p>Stop the YARN session by stopping the unix process (using CTRL+C) or by entering âstopâ into the client.</p> - -<h4 id="detached-yarn-session">Detached YARN session</h4> - -<p>If you do not want to keep the Flink YARN client running all the time, its also possible to start a <em>detached</em> YARN session. -The parameter for that is called <code>-d</code> or <code>--detached</code>.</p> - -<p>In that case, the Flink YARN client will only submit Flink to the cluster and then close itself. -Note that in this case its not possible to stop the YARN session using Flink.</p> - -<p>Use the YARN utilities (<code>yarn application -kill <appId</code>) to stop the YARN session.</p> - -<h3 id="submit-job-to-flink">Submit Job to Flink</h3> - -<p>Use the following command to submit a Flink program to the YARN cluster:</p> - -<div class="highlight"><pre><code class="language-bash">./bin/flink</code></pre></div> - -<p>Please refer to the documentation of the <a href="http://flink.apache.org/docs/0.9/apis/cli.html";>commandline client</a>.</p> - -<p>The command will show you a help menu like this:</p> - -<div class="highlight"><pre><code class="language-bash"><span class="o">[</span>...<span class="o">]</span> -Action <span class="s2">"run"</span> compiles and runs a program. - - Syntax: run <span class="o">[</span>OPTIONS<span class="o">]</span> <jar-file> <arguments> - <span class="s2">"run"</span> action arguments: - -c,--class <classname> Class with the program entry point <span class="o">(</span><span class="s2">"main"</span> - method or <span class="s2">"getPlan()"</span> method. Only needed - <span class="k">if</span> the JAR file does not specify the class - in its manifest. - -m,--jobmanager <host:port> Address of the JobManager <span class="o">(</span>master<span class="o">)</span> to - which to connect. Use this flag to connect - to a different JobManager than the one - specified in the configuration. - -p,--parallelism <parallelism> The parallelism with which to run the - program. Optional flag to override the - default value specified in the - configuration</code></pre></div> - -<p>Use the <em>run</em> action to submit a job to YARN. The client is able to determine the address of the JobManager. In the rare event of a problem, you can also pass the JobManager address using the <code>-m</code> argument. The JobManager address is visible in the YARN console.</p> - -<p><strong>Example</strong></p> - -<div class="highlight"><pre><code class="language-bash">wget -O apache-license-v2.txt http://www.apache.org/licenses/LICENSE-2.0.txt -hadoop fs -copyFromLocal LICENSE-2.0.txt hdfs:/// ... -./bin/flink run ./examples/flink-java-examples-0.9.0-WordCount.jar <span class="se">\</span> - hdfs:///..../apache-license-v2.txt hdfs:///.../wordcount-result.txt</code></pre></div> - -<p>If there is the following error, make sure that all TaskManagers started:</p> - -<div class="highlight"><pre><code class="language-bash">Exception in thread <span class="s2">"main"</span> org.apache.flink.compiler.CompilerException: - Available instances could not be determined from job manager: Connection timed out.</code></pre></div> - -<p>You can check the number of TaskManagers in the JobManager web interface. The address of this interface is printed in the YARN session console.</p> - -<p>If the TaskManagers do not show up after a minute, you should investigate the issue using the log files.</p> - -<h2 id="run-a-single-flink-job-on-hadoop-yarn">Run a single Flink job on Hadoop YARN</h2> - -<p>The documentation above describes how to start a Flink cluster within a Hadoop YARN environment. -It is also possible to launch Flink within YARN only for executing a single job.</p> - -<p>Please note that the client then expects the <code>-yn</code> value to be set (number of TaskManagers).</p> - -<p><strong><em>Example:</em></strong></p> - -<div class="highlight"><pre><code class="language-bash">./bin/flink run -m yarn-cluster -yn <span class="m">2</span> ./examples/flink-java-examples-0.9.0-WordCount.jar</code></pre></div> - -<p>The command line options of the YARN session are also available with the <code>./bin/flink</code> tool. They are prefixed with a <code>y</code> or <code>yarn</code> (for the long argument options).</p> - -<h2 id="recovery-behavior-of-flink-on-yarn">Recovery behavior of Flink on YARN</h2> - -<p>Flinkâs YARN client has the following configuration parameters to control how to behave in case of container failures. These parameters can be set either from the <code>conf/flink-conf.yaml</code> or when starting the YARN session, using <code>-D</code> parameters.</p> - -<ul> - <li><code>yarn.reallocate-failed</code>: This parameter controls whether Flink should reallocate failed TaskManager containers. Default: true</li> - <li><code>yarn.maximum-failed-containers</code>: The maximum number of failed containers the ApplicationMaster accepts until it fails the YARN session. Default: The number of initally requested TaskManagers (<code>-n</code>).</li> - <li><code>yarn.application-attempts</code>: The number of ApplicationMaster (+ its TaskManager containers) attempts. If this value is set to 1 (default), the entire YARN session will fail when the Application master fails. Higher values specify the number of restarts of the ApplicationMaster by YARN.</li> -</ul> - -<h2 id="debugging-a-failed-yarn-session">Debugging a failed YARN session</h2> - -<p>There are many reasons why a Flink YARN session deployment can fail. A misconfigured Hadoop setup (HDFS permissions, YARN configuration), version incompatibilities (running Flink with vanilla Hadoop dependencies on Cloudera Hadoop) or other errors.</p> - -<h3 id="log-files">Log Files</h3> - -<p>In cases where the Flink YARN session fails during the deployment itself, users have to rely on the logging capabilities of Hadoop YARN. The most useful feature for that is the <a href="http://hortonworks.com/blog/simplifying-user-logs-management-and-access-in-yarn/";>YARN log aggregation</a>. -To enable it, users have to set the <code>yarn.log-aggregation-enable</code> property to <code>true</code> in the <code>yarn-site.xml</code> file. -Once that is enabled, users can use the following command to retrieve all log files of a (failed) YARN session.</p> - -<div class="highlight"><pre><code>yarn logs -applicationId <application ID> -</code></pre></div> - -<p>Note that it takes a few seconds after the session has finished until the logs show up.</p> - -<h3 id="yarn-client-console--webinterfaces">YARN Client console & Webinterfaces</h3> - -<p>The Flink YARN client also prints error messages in the terminal if errors occur during runtime (for example if a TaskManager stops working after some time).</p> - -<p>In addition to that, there is the YARN Resource Manager webinterface (by default on port 8088). The port of the Resource Manager web interface is determined by the <code>yarn.resourcemanager.webapp.address</code> configuration value.</p> - -<p>It allows to access log files for running YARN applications and shows diagnostics for failed apps.</p> - -<h2 id="build-yarn-client-for-a-specific-hadoop-version">Build YARN client for a specific Hadoop version</h2> - -<p>Users using Hadoop distributions from companies like Hortonworks, Cloudera or MapR might have to build Flink against their specific versions of Hadoop (HDFS) and YARN. Please read the <a href="building.html">build instructions</a> for more details.</p> - -<h2 id="background--internals">Background / Internals</h2> - -<p>This section briefly describes how Flink and YARN interact.</p> - -<p><img src="fig/FlinkOnYarn.svg" class="img-responsive" /></p> - -<p>The YARN client needs to access the Hadoop configuration to connect to the YARN resource manager and to HDFS. It determines the Hadoop configuration using the following strategy:</p> - -<ul> - <li>Test if <code>YARN_CONF_DIR</code>, <code>HADOOP_CONF_DIR</code> or <code>HADOOP_CONF_PATH</code> are set (in that order). If one of these variables are set, they are used to read the configuration.</li> - <li>If the above strategy fails (this should not be the case in a correct YARN setup), the client is using the <code>HADOOP_HOME</code> environment variable. If it is set, the client tries to access <code>$HADOOP_HOME/etc/hadoop</code> (Hadoop 2) and <code>$HADOOP_HOME/conf</code> (Hadoop 1).</li> -</ul> - -<p>When starting a new Flink YARN session, the client first checks if the requested resources (containers and memory) are available. After that, it uploads a jar that contains Flink and the configuration to HDFS (step 1).</p> - -<p>The next step of the client is to request (step 2) a YARN container to start the <em>ApplicationMaster</em> (step 3). Since the client registered the configuration and jar-file as a resource for the container, the NodeManager of YARN running on that particular machine will take care of preparing the container (e.g. downloading the files). Once that has finished, the <em>ApplicationMaster</em> (AM) is started.</p> - -<p>The <em>JobManager</em> and AM are running in the same container. Once they successfully started, the AM knows the address of the JobManager (its own host). It is generating a new Flink configuration file for the TaskManagers (so that they can connect to the JobManager). The file is also uploaded to HDFS. Additionally, the <em>AM</em> container is also serving Flinkâs web interface. The ports Flink is using for its services are the standard ports configured by the user + the application id as an offset. This allows users to execute multiple Flink YARN sessions in parallel.</p> - -<p>After that, the AM starts allocating the containers for Flinkâs TaskManagers, which will download the jar file and the modified configuration from the HDFS. Once these steps are completed, Flink is set up and ready to accept Jobs.</p> - - - </div> - - <div class="col-sm-10 col-sm-offset-1"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> -</div> - - </div><!-- /.container --> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js";></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - - <!-- Disqus --> - <script type="text/javascript"> - var disqus_shortname = 'stratosphere-eu'; - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); -</script> - </body> -</html>
http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/master/README.md ---------------------------------------------------------------------- diff --git a/content/docs/master/README.md b/content/docs/master/README.md deleted file mode 100644 index 206b04c..0000000 --- a/content/docs/master/README.md +++ /dev/null @@ -1,60 +0,0 @@ -This README gives an overview of how to build and contribute to the documentation of Apache Flink. - -The documentation is included with the source of Apache Flink in order to ensure that you always -have docs corresponding to your checked out version. The online documentation at -http://flink.apache.org/ is also generated from the files found here. - -# Requirements - -We use Markdown to write and Jekyll to translate the documentation to static HTML. You can install -all needed software via: - - gem install jekyll - gem install kramdown - sudo easy_install Pygments - -Kramdown is needed for Markdown processing and the Python based Pygments is used for syntax -highlighting. - -# Build - -The `docs/build_docs.sh` script calls Jekyll and generates the documentation in `docs/target`. You -can then point your browser to `docs/target/index.html` and start reading. - -If you call the script with the preview flag `build_docs.sh -p`, Jekyll will start a web server at -`localhost:4000` and watch the docs directory for updates. Use this mode to preview changes locally. - -# Contribute - -The documentation pages are written in -[Markdown](http://daringfireball.net/projects/markdown/syntax). It is possible to use the -[GitHub flavored syntax](http://github.github.com/github-flavored-markdown) and intermix plain html. - -In addition to Markdown, every page contains a Jekyll front matter, which specifies the title of the -page and the layout to use. The title is used as the top-level heading for the page. - - --- - title: "Title of the Page" - --- - -Furthermore, you can access variables found in `docs/_config.yml` as follows: - - {{ site.NAME }} - -This will be replaced with the value of the variable called `NAME` when generating -the docs. - -All documents are structed with headings. From these heading, a page outline is -automatically generated for each page. - -``` -# Level-1 Heading <- Used for the title of the page -## Level-2 Heading <- Start with this one -### Level-3 heading -#### Level-4 heading -##### Level-5 heading -``` - -Please stick to the "logical order" when using the headlines, e.g. start with level-2 headings and -use level-3 headings for subsections, etc. Don't use a different ordering, because you don't like -how a headline looks. http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/master/apis/best_practices.html ---------------------------------------------------------------------- diff --git a/content/docs/master/apis/best_practices.html b/content/docs/master/apis/best_practices.html deleted file mode 100644 index 44b220b..0000000 --- a/content/docs/master/apis/best_practices.html +++ /dev/null @@ -1,411 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<!DOCTYPE html> - -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - - <title>Apache Flink 0.10-SNAPSHOT Documentation: Best Practices</title> - - <link rel="shortcut icon" href="http://flink.apache.org/docs/master/page/favicon.ico"; type="image/x-icon"> - <link rel="icon" href="http://flink.apache.org/docs/master/page/favicon.ico"; type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/flink.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/syntax.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/codetabs.css";> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - - - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="http://flink.apache.org/docs/master/page/img/navbar-brand-logo.jpg";></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li><a href="http://flink.apache.org/docs/master/index.html";>Overview<span class="hidden-sm hidden-xs"> 0.10</span></a></li> - - <!-- Setup --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/setup"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/setup/building.html";>Get Flink 0.10-SNAPSHOT</a></li> - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - <li><a href="http://flink.apache.org/docs/master/setup/local_setup.html"; class="active">Local</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/cluster_setup.html";>Cluster (Standalone)</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/yarn_setup.html";>YARN</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/gce_setup.html";>GCloud</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/flink_on_tez.html";>Flink on Tez <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="http://flink.apache.org/docs/master/setup/config.html";>Configuration</a></li> - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/apis"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/apis/programming_guide.html";><strong>Batch: DataSet API</strong></a></li> - <li><a href="http://flink.apache.org/docs/master/apis/streaming_guide.html";><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/apis/python.html";>Python API <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="scala_shell.html">Interactive Scala Shell</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/dataset_transformations.html";>Dataset Transformations</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/best_practices.html";>Best Practices</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/example_connectors.html";>Connectors</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/examples.html";>Examples</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/local_execution.html";>Local Execution</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/cluster_execution.html";>Cluster Execution</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/cli.html";>Command Line Interface</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/web_client.html";>Web Client</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/iterations.html";>Iterations</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/java8.html";>Java 8</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/hadoop_compatibility.html";>Hadoop Compatability <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/libs"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/libs/spargel_guide.html";>Graphs: Spargel</a></li> - <li><a href="http://flink.apache.org/docs/master/libs/gelly_guide.html";>Graphs: Gelly <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/libs/ml/";>Machine Learning <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/libs/table.html";>Relational: Table <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Internals --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/internals"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/docs/master/internals/how_to_contribute.html";>How to Contribute</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/coding_guidelines.html";>Coding Guidelines</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/ide_setup.html";>IDE Setup</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/logging.html";>Logging</a></li> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Internals</strong></li> - <li><a href="http://flink.apache.org/docs/master/internals/general_arch.html";>Architecture & Process Model</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/types_serialization.html";>Type Extraction & Serialization</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/job_scheduling.html";>Jobs & Scheduling</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/add_operator.html";>How-To: Add an Operator</a></li> - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/master/search-results.html";> - <div class="form-group"> - <input type="text" class="form-control" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-10 col-sm-offset-1"> - <h1>Best Practices</h1> - - - -<p><a href="#top"></a></p> - -<p>This page contains a collection of best practices for Flink programmers on how to solve frequently encountered problems.</p> - -<ul id="markdown-toc"> - <li><a href="#parsing-command-line-arguments-and-passing-them-around-in-your-flink-application" id="markdown-toc-parsing-command-line-arguments-and-passing-them-around-in-your-flink-application">Parsing command line arguments and passing them around in your Flink application</a> <ul> - <li><a href="#getting-your-configuration-values-into-the-parametertool" id="markdown-toc-getting-your-configuration-values-into-the-parametertool">Getting your configuration values into the <code>ParameterTool</code></a></li> - <li><a href="#using-the-parameters-in-your-flink-program" id="markdown-toc-using-the-parameters-in-your-flink-program">Using the parameters in your Flink program</a></li> - </ul> - </li> - <li><a href="#naming-large-tuplex-types" id="markdown-toc-naming-large-tuplex-types">Naming large TupleX types</a></li> - <li><a href="#register-a-custom-serializer-for-your-flink-program" id="markdown-toc-register-a-custom-serializer-for-your-flink-program">Register a custom serializer for your Flink program</a></li> -</ul> - -<h2 id="parsing-command-line-arguments-and-passing-them-around-in-your-flink-application">Parsing command line arguments and passing them around in your Flink application</h2> - -<p>Almost all Flink applications, both batch and streaming rely on external configuration parameters. -For example for specifying input and output sources (like paths or addresses), also system parameters (parallelism, runtime configuration) and application specific parameters (often used within the user functions).</p> - -<p>Since version 0.9 we are providing a simple utility called <code>ParameterTool</code> to provide at least some basic tooling for solving these problems.</p> - -<p>Please note that you donât have to use the <code>ParameterTool</code> explained here. Other frameworks such as <a href="https://commons.apache.org/proper/commons-cli/";>Commons CLI</a>, -<a href="http://argparse4j.sourceforge.net/";>argparse4j</a> and others work well with Flink as well.</p> - -<h3 id="getting-your-configuration-values-into-the-parametertool">Getting your configuration values into the <code>ParameterTool</code></h3> - -<p>The <code>ParameterTool</code> provides a set of predefined static methods for reading the configuration. The tool is internally expecting a <code>Map<String, String></code>, so its very easy to integrate it with your own configuration style.</p> - -<h4 id="from-properties-files">From <code>.properties</code> files</h4> - -<p>The following method will read a <a href="https://docs.oracle.com/javase/tutorial/essential/environment/properties.html";>Properties</a> file and provide the key/value pairs:</p> - -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">String</span> <span class="n">propertiesFile</span> <span class="o">=</span> <span class="s">"/home/sam/flink/myjob.properties"</span><span class="o">;</span> -<span class="n">ParameterTool</span> <span class="n">parameter</span> <span class="o">=</span> <span class="n">ParameterTool</span><span class="o">.</span><span class="na">fromPropertiesFile</span><span class="o">(</span><span class="n">propertiesFile</span><span class="o">);</span></code></pre></div> - -<h4 id="from-the-command-line-arguments">From the command line arguments</h4> - -<p>This allows getting arguments like <code>--input hdfs:///mydata --elements 42</code> from the command line.</p> - -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="nf">main</span><span class="o">(</span><span class="n">String</span><span class="o">[]</span> <span class="n">args</span><span class="o">)</span> <span class="o">{</span> - <span class="n">ParameterTool</span> <span class="n">parameter</span> <span class="o">=</span> <span class="n">ParameterTool</span><span class="o">.</span><span class="na">fromArgs</span><span class="o">(</span><span class="n">args</span><span class="o">);</span> - <span class="c1">// .. regular code ..</span></code></pre></div> - -<h4 id="from-system-properties">From system properties</h4> - -<p>When starting a JVM, you can pass system properties to it: <code>-Dinput=hdfs:///mydata</code>. You can also initialize the <code>ParameterTool</code> from these system properties:</p> - -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">ParameterTool</span> <span class="n">parameter</span> <span class="o">=</span> <span class="n">ParameterTool</span><span class="o">.</span><span class="na">fromSystemProperties</span><span class="o">();</span></code></pre></div> - -<h3 id="using-the-parameters-in-your-flink-program">Using the parameters in your Flink program</h3> - -<p>Now that weâve got the parameters from somewhere (see above) we can use them in various ways.</p> - -<p><strong>Directly from the <code>ParameterTool</code></strong></p> - -<p>The <code>ParameterTool</code> itself has methods for accessing the values.</p> - -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">ParameterTool</span> <span class="n">parameters</span> <span class="o">=</span> <span class="c1">// ...</span> -<span class="n">parameter</span><span class="o">.</span><span class="na">getRequired</span><span class="o">(</span><span class="s">"input"</span><span class="o">);</span> -<span class="n">parameter</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="s">"output"</span><span class="o">,</span> <span class="s">"myDefaultValue"</span><span class="o">);</span> -<span class="n">parameter</span><span class="o">.</span><span class="na">getLong</span><span class="o">(</span><span class="s">"expectedCount"</span><span class="o">,</span> <span class="o">-</span><span class="mi">1L</span><span class="o">);</span> -<span class="n">parameter</span><span class="o">.</span><span class="na">getNumberOfParameters</span><span class="o">()</span> -<span class="c1">// .. there are more methods available.</span></code></pre></div> - -<p>You can use the return values of these methods directly in the main() method (=the client submitting the application). -For example you could set the parallelism of a operator like this:</p> - -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">ParameterTool</span> <span class="n">parameters</span> <span class="o">=</span> <span class="n">ParameterTool</span><span class="o">.</span><span class="na">fromArgs</span><span class="o">(</span><span class="n">args</span><span class="o">);</span> -<span class="kt">int</span> <span class="n">parallelism</span> <span class="o">=</span> <span class="n">parameters</span><span class="o">.</span><span class="na">get</span><span class="o">(</span><span class="s">"mapParallelism"</span><span class="o">,</span> <span class="mi">2</span><span class="o">);</span> -<span class="n">DataSet</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Integer</span><span class="o">>></span> <span class="n">counts</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="na">flatMap</span><span class="o">(</span><span class="k">new</span> <span class="nf">Tokenizer</span><span class="o">()).</span><span class="na">setParallelism</span><span class="o">(</span><span class="n">parallelism</span><span class="o">);</span></code></pre></div> - -<p>Since the <code>ParameterTool</code> is serializable, you can pass it to the functions itself:</p> - -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">ParameterTool</span> <span class="n">parameters</span> <span class="o">=</span> <span class="n">ParameterTool</span><span class="o">.</span><span class="na">fromArgs</span><span class="o">(</span><span class="n">args</span><span class="o">);</span> -<span class="n">DataSet</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Integer</span><span class="o">>></span> <span class="n">counts</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="na">flatMap</span><span class="o">(</span><span class="k">new</span> <span class="nf">Tokenizer</span><span class="o">(</span><span class="n">parameters</span><span class="o">));</span></code></pre></div> - -<p>and then use them inside the function for getting values from the command line.</p> - -<h4 id="passing-it-as-a-configuration-object-to-single-functions">Passing it as a <code>Configuration</code> object to single functions</h4> - -<p>The example below shows how to pass the parameters as a <code>Configuration</code> object to a user defined function.</p> - -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">ParameterTool</span> <span class="n">parameters</span> <span class="o">=</span> <span class="n">ParameterTool</span><span class="o">.</span><span class="na">fromArgs</span><span class="o">(</span><span class="n">args</span><span class="o">);</span> -<span class="n">DataSet</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Integer</span><span class="o">>></span> <span class="n">counts</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="na">flatMap</span><span class="o">(</span><span class="k">new</span> <span class="nf">Tokenizer</span><span class="o">()).</span><span class="na">withParameters</span><span class="o">(</span><span class="n">parameters</span><span class="o">.</span><span class="na">getConfiguration</span><span class="o">())</span></code></pre></div> - -<p>In the <code>Tokenizer</code>, the object is now accessible in the <code>open(Configuration conf)</code> method:</p> - -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">static</span> <span class="kd">final</span> <span class="kd">class</span> <span class="nc">Tokenizer</span> <span class="kd">extends</span> <span class="n">RichFlatMapFunction</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Tuple2</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Integer</span><span class="o">>></span> <span class="o">{</span> - <span class="nd">@Override</span> - <span class="kd">public</span> <span class="kt">void</span> <span class="nf">open</span><span class="o">(</span><span class="n">Configuration</span> <span class="n">parameters</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="o">{</span> - <span class="n">parameters</span><span class="o">.</span><span class="na">getInteger</span><span class="o">(</span><span class="s">"myInt"</span><span class="o">,</span> <span class="o">-</span><span class="mi">1</span><span class="o">);</span> - <span class="c1">// .. do</span></code></pre></div> - -<h4 id="register-the-parameters-globally">Register the parameters globally</h4> - -<p>Parameters registered as a <a href="programming_guide.html#passing-parameters-to-functions">global job parameter</a> at the <code>ExecutionConfig</code> allow you to access the configuration values from the JobManager web interface and all functions defined by the user.</p> - -<p><strong>Register the parameters globally</strong></p> - -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">ParameterTool</span> <span class="n">parameters</span> <span class="o">=</span> <span class="n">ParameterTool</span><span class="o">.</span><span class="na">fromArgs</span><span class="o">(</span><span class="n">args</span><span class="o">);</span> - -<span class="c1">// set up the execution environment</span> -<span class="kd">final</span> <span class="n">ExecutionEnvironment</span> <span class="n">env</span> <span class="o">=</span> <span class="n">ExecutionEnvironment</span><span class="o">.</span><span class="na">getExecutionEnvironment</span><span class="o">();</span> -<span class="n">env</span><span class="o">.</span><span class="na">getConfig</span><span class="o">().</span><span class="na">setGlobalJobParameters</span><span class="o">(</span><span class="n">parameters</span><span class="o">);</span></code></pre></div> - -<p>Access them in any rich user function:</p> - -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">static</span> <span class="kd">final</span> <span class="kd">class</span> <span class="nc">Tokenizer</span> <span class="kd">extends</span> <span class="n">RichFlatMapFunction</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Tuple2</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Integer</span><span class="o">>></span> <span class="o">{</span> - - <span class="nd">@Override</span> - <span class="kd">public</span> <span class="kt">void</span> <span class="nf">flatMap</span><span class="o">(</span><span class="n">String</span> <span class="n">value</span><span class="o">,</span> <span class="n">Collector</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Integer</span><span class="o">>></span> <span class="n">out</span><span class="o">)</span> <span class="o">{</span> - <span class="n">ParameterTool</span> <span class="n">parameters</span> <span class="o">=</span> <span class="o">(</span><span class="n">ParameterTool</span><span class="o">)</span> <span class="n">getRuntimeContext</span><span class="o">().</span><span class="na">getExecutionConfig</span><span class="o">().</span><span class="na">getGlobalJobParameters</span><span class="o">();</span> - <span class="n">parameters</span><span class="o">.</span><span class="na">getRequired</span><span class="o">(</span><span class="s">"input"</span><span class="o">);</span> - <span class="c1">// .. do more ..</span></code></pre></div> - -<h2 id="naming-large-tuplex-types">Naming large TupleX types</h2> - -<p>It is recommended to use POJOs (Plain old Java objects) instead of <code>TupleX</code> for data types with many fields. -Also, POJOs can be used to give large <code>Tuple</code>-types a name.</p> - -<p><strong>Example</strong></p> - -<p>Instead of using:</p> - -<div class="highlight"><pre><code class="language-java"><span class="n">Tuple11</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">String</span><span class="o">,</span> <span class="o">...,</span> <span class="n">String</span><span class="o">></span> <span class="n">var</span> <span class="o">=</span> <span class="k">new</span> <span class="o">...;</span></code></pre></div> - -<p>It is much easier to create a custom type extending from the large Tuple type.</p> - -<div class="highlight"><pre><code class="language-java"><span class="n">CustomType</span> <span class="n">var</span> <span class="o">=</span> <span class="k">new</span> <span class="o">...;</span> - -<span class="kd">public</span> <span class="kd">static</span> <span class="kd">class</span> <span class="nc">CustomType</span> <span class="kd">extends</span> <span class="n">Tuple11</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">String</span><span class="o">,</span> <span class="o">...,</span> <span class="n">String</span><span class="o">></span> <span class="o">{</span> - <span class="c1">// constructor matching super</span> -<span class="o">}</span></code></pre></div> - -<h2 id="register-a-custom-serializer-for-your-flink-program">Register a custom serializer for your Flink program</h2> - -<p>If you use a custom type in your Flink program which cannot be serialized by the -Flink type serializer, Flink falls back to using the generic Kryo -serializer. You may register your own serializer or a serialization system like -Google Protobuf or Apache Thrift with Kryo. To do that, simply register the type -class and the serializer in the <code>ExecutionConfig</code> of your Flink program.</p> - -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">final</span> <span class="n">ExecutionEnvironment</span> <span class="n">env</span> <span class="o">=</span> <span class="n">ExecutionEnvironment</span><span class="o">.</span><span class="na">getExecutionEnvironment</span><span class="o">();</span> - -<span class="c1">// register the class of the serializer as serializer for a type</span> -<span class="n">env</span><span class="o">.</span><span class="na">getConfig</span><span class="o">().</span><span class="na">registerTypeWithKryoSerializer</span><span class="o">(</span><span class="n">MyCustomType</span><span class="o">.</span><span class="na">class</span><span class="o">,</span> <span class="n">MyCustomSerializer</span><span class="o">.</span><span class="na">class</span><span class="o">);</span> - -<span class="c1">// register an instance as serializer for a type</span> -<span class="n">MySerializer</span> <span class="n">mySerializer</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">MySerializer</span><span class="o">();</span> -<span class="n">env</span><span class="o">.</span><span class="na">getConfig</span><span class="o">().</span><span class="na">registerTypeWithKryoSerializer</span><span class="o">(</span><span class="n">MyCustomType</span><span class="o">.</span><span class="na">class</span><span class="o">,</span> <span class="n">mySerializer</span><span class="o">);</span></code></pre></div> - -<p>Note that your custom serializer has to extend Kryoâs Serializer class. In the -case of Google Protobuf or Apache Thrift, this has already been done for -you:</p> - -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">final</span> <span class="n">ExecutionEnvironment</span> <span class="n">env</span> <span class="o">=</span> <span class="n">ExecutionEnvironment</span><span class="o">.</span><span class="na">getExecutionEnvironment</span><span class="o">();</span> - -<span class="c1">// register the Google Protobuf serializer with Kryo</span> -<span class="n">env</span><span class="o">.</span><span class="na">getConfig</span><span class="o">().</span><span class="na">registerTypeWithKryoSerializer</span><span class="o">(</span><span class="n">MyCustomType</span><span class="o">.</span><span class="na">class</span><span class="o">,</span> <span class="n">ProtobufSerializer</span><span class="o">.</span><span class="na">class</span><span class="o">);</span> - -<span class="c1">// register the serializer included with Apache Thrift as the standard serializer</span> -<span class="c1">// TBaseSerializer states it should be initalized as a default Kryo serializer</span> -<span class="n">env</span><span class="o">.</span><span class="na">getConfig</span><span class="o">().</span><span class="na">addDefaultKryoSerializer</span><span class="o">(</span><span class="n">MyCustomType</span><span class="o">.</span><span class="na">class</span><span class="o">,</span> <span class="n">TBaseSerializer</span><span class="o">.</span><span class="na">class</span><span class="o">);</span></code></pre></div> - -<p>For the above example to work, you need to include the necessary dependencies in -your Maven project file (pom.xml). In the dependency section, add the following -for Apache Thrift:</p> - -<div class="highlight"><pre><code class="language-xml" data-lang="xml"><span class="nt"><dependency></span> - <span class="nt"><groupId></span>com.twitter<span class="nt"></groupId></span> - <span class="nt"><artifactId></span>chill-thrift<span class="nt"></artifactId></span> - <span class="nt"><version></span>0.5.2<span class="nt"></version></span> -<span class="nt"></dependency></span> -<span class="c"><!-- libthrift is required by chill-thrift --></span> -<span class="nt"><dependency></span> - <span class="nt"><groupId></span>org.apache.thrift<span class="nt"></groupId></span> - <span class="nt"><artifactId></span>libthrift<span class="nt"></artifactId></span> - <span class="nt"><version></span>0.6.1<span class="nt"></version></span> - <span class="nt"><exclusions></span> - <span class="nt"><exclusion></span> - <span class="nt"><groupId></span>javax.servlet<span class="nt"></groupId></span> - <span class="nt"><artifactId></span>servlet-api<span class="nt"></artifactId></span> - <span class="nt"></exclusion></span> - <span class="nt"><exclusion></span> - <span class="nt"><groupId></span>org.apache.httpcomponents<span class="nt"></groupId></span> - <span class="nt"><artifactId></span>httpclient<span class="nt"></artifactId></span> - <span class="nt"></exclusion></span> - <span class="nt"></exclusions></span> -<span class="nt"></dependency></span></code></pre></div> - -<p>For Google Protobuf you need the following Maven dependency:</p> - -<div class="highlight"><pre><code class="language-xml" data-lang="xml"><span class="nt"><dependency></span> - <span class="nt"><groupId></span>com.twitter<span class="nt"></groupId></span> - <span class="nt"><artifactId></span>chill-protobuf<span class="nt"></artifactId></span> - <span class="nt"><version></span>0.5.2<span class="nt"></version></span> -<span class="nt"></dependency></span> -<span class="c"><!-- We need protobuf for chill-protobuf --></span> -<span class="nt"><dependency></span> - <span class="nt"><groupId></span>com.google.protobuf<span class="nt"></groupId></span> - <span class="nt"><artifactId></span>protobuf-java<span class="nt"></artifactId></span> - <span class="nt"><version></span>2.5.0<span class="nt"></version></span> -<span class="nt"></dependency></span></code></pre></div> - -<p>Please adjust the versions of both libraries as needed.</p> - - </div> - - <div class="col-sm-10 col-sm-offset-1"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> -</div> - - </div><!-- /.container --> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="http://flink.apache.org/docs/master/page/js/codetabs.js";></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - - <!-- Disqus --> - <script type="text/javascript"> - var disqus_shortname = 'stratosphere-eu'; - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); -</script> - </body> -</html> http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/master/apis/cli.html ---------------------------------------------------------------------- diff --git a/content/docs/master/apis/cli.html b/content/docs/master/apis/cli.html deleted file mode 100644 index bfe695e..0000000 --- a/content/docs/master/apis/cli.html +++ /dev/null @@ -1,394 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<!DOCTYPE html> - -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - - <title>Apache Flink 0.10-SNAPSHOT Documentation: Command-Line Interface</title> - - <link rel="shortcut icon" href="http://flink.apache.org/docs/master/page/favicon.ico"; type="image/x-icon"> - <link rel="icon" href="http://flink.apache.org/docs/master/page/favicon.ico"; type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/flink.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/syntax.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/codetabs.css";> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - - - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="http://flink.apache.org/docs/master/page/img/navbar-brand-logo.jpg";></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li><a href="http://flink.apache.org/docs/master/index.html";>Overview<span class="hidden-sm hidden-xs"> 0.10</span></a></li> - - <!-- Setup --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/setup"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/setup/building.html";>Get Flink 0.10-SNAPSHOT</a></li> - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - <li><a href="http://flink.apache.org/docs/master/setup/local_setup.html"; class="active">Local</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/cluster_setup.html";>Cluster (Standalone)</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/yarn_setup.html";>YARN</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/gce_setup.html";>GCloud</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/flink_on_tez.html";>Flink on Tez <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="http://flink.apache.org/docs/master/setup/config.html";>Configuration</a></li> - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/apis"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/apis/programming_guide.html";><strong>Batch: DataSet API</strong></a></li> - <li><a href="http://flink.apache.org/docs/master/apis/streaming_guide.html";><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/apis/python.html";>Python API <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="scala_shell.html">Interactive Scala Shell</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/dataset_transformations.html";>Dataset Transformations</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/best_practices.html";>Best Practices</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/example_connectors.html";>Connectors</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/examples.html";>Examples</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/local_execution.html";>Local Execution</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/cluster_execution.html";>Cluster Execution</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/cli.html";>Command Line Interface</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/web_client.html";>Web Client</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/iterations.html";>Iterations</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/java8.html";>Java 8</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/hadoop_compatibility.html";>Hadoop Compatability <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/libs"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/libs/spargel_guide.html";>Graphs: Spargel</a></li> - <li><a href="http://flink.apache.org/docs/master/libs/gelly_guide.html";>Graphs: Gelly <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/libs/ml/";>Machine Learning <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/libs/table.html";>Relational: Table <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Internals --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/internals"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/docs/master/internals/how_to_contribute.html";>How to Contribute</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/coding_guidelines.html";>Coding Guidelines</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/ide_setup.html";>IDE Setup</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/logging.html";>Logging</a></li> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Internals</strong></li> - <li><a href="http://flink.apache.org/docs/master/internals/general_arch.html";>Architecture & Process Model</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/types_serialization.html";>Type Extraction & Serialization</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/job_scheduling.html";>Jobs & Scheduling</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/add_operator.html";>How-To: Add an Operator</a></li> - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/master/search-results.html";> - <div class="form-group"> - <input type="text" class="form-control" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-10 col-sm-offset-1"> - <h1>Command-Line Interface</h1> - - - -<p>Flink provides a command-line interface to run programs that are packaged -as JAR files, and control their execution. The command line interface is part -of any Flink setup, available in local single node setups and in -distributed setups. It is located under <code><flink-home>/bin/flink</code> -and connects by default to the running Flink master (JobManager) that was -started from the same installation directory.</p> - -<p>A prerequisite to using the command line interface is that the Flink -master (JobManager) has been started (via <code><flink-home>/bin/start- -local.sh</code> or <code><flink-home>/bin/start-cluster.sh</code>) or that a YARN -environment is available.</p> - -<p>The command line can be used to</p> - -<ul id="markdown-toc"> - <li><a href="#examples" id="markdown-toc-examples">Examples</a></li> - <li><a href="#usage" id="markdown-toc-usage">Usage</a></li> -</ul> - -<h2 id="examples">Examples</h2> - -<ul> - <li> - <p>Run example program with no arguments.</p> - - <div class="highlight"><pre><code>./bin/flink run ./examples/flink-java-examples-0.10-SNAPSHOT-WordCount.jar -</code></pre></div> - </li> - <li> - <p>Run example program with arguments for input and result files</p> - - <div class="highlight"><pre><code>./bin/flink run ./examples/flink-java-examples-0.10-SNAPSHOT-WordCount.jar \ - file:///home/user/hamlet.txt file:///home/user/wordcount_out -</code></pre></div> - </li> - <li> - <p>Run example program with parallelism 16 and arguments for input and result files</p> - - <div class="highlight"><pre><code>./bin/flink run -p 16 ./examples/flink-java-examples-0.10-SNAPSHOT-WordCount.jar \ - file:///home/user/hamlet.txt file:///home/user/wordcount_out -</code></pre></div> - </li> - <li> - <p>Run example program on a specific JobManager:</p> - - <div class="highlight"><pre><code>./bin/flink run -m myJMHost:6123 \ - ./examples/flink-java-examples-0.10-SNAPSHOT-WordCount.jar \ - file:///home/user/hamlet.txt file:///home/user/wordcount_out -</code></pre></div> - </li> - <li> - <p>Run example program with a specific class as an entry point:</p> - - <div class="highlight"><pre><code>./bin/flink run -c org.apache.flink.examples.java.wordcount.WordCount \ - ./examples/flink-java-examples-0.10-SNAPSHOT-WordCount.jar \ - file:///home/user/hamlet.txt file:///home/user/wordcount_out -</code></pre></div> - </li> - <li> - <p>Run example program using a <a href="http://flink.apache.org/docs/master/setup/yarn_setup.html#run-a-single-flink-job-on-hadoop-yarn";>per-job YARN cluster</a> with 2 TaskManagers:</p> - - <div class="highlight"><pre><code>./bin/flink run -m yarn-cluster -yn 2 \ - ./examples/flink-java-examples-0.10-SNAPSHOT-WordCount.jar \ - hdfs:///user/hamlet.txt hdfs:///user/wordcount_out -</code></pre></div> - </li> - <li> - <p>Display the optimized execution plan for the WordCount example program as JSON:</p> - - <div class="highlight"><pre><code>./bin/flink info ./examples/flink-java-examples-0.10-SNAPSHOT-WordCount.jar \ - file:///home/user/hamlet.txt file:///home/user/wordcount_out -</code></pre></div> - </li> - <li> - <p>List scheduled and running jobs (including their JobIDs):</p> - - <div class="highlight"><pre><code>./bin/flink list -</code></pre></div> - </li> - <li> - <p>List scheduled jobs (including their JobIDs):</p> - - <div class="highlight"><pre><code>./bin/flink list -s -</code></pre></div> - </li> - <li> - <p>List running jobs (including their JobIDs):</p> - - <div class="highlight"><pre><code>./bin/flink list -r -</code></pre></div> - </li> - <li> - <p>Cancel a job:</p> - - <div class="highlight"><pre><code>./bin/flink cancel <jobID> -</code></pre></div> - </li> -</ul> - -<h2 id="usage">Usage</h2> - -<p>The command line syntax is as follows:</p> - -<div class="highlight"><pre><code>./flink <ACTION> [OPTIONS] [ARGUMENTS] - -The following actions are available: - -Action "run" compiles and runs a program. - - Syntax: run [OPTIONS] <jar-file> <arguments> - "run" action options: - -c,--class <classname> Class with the program entry point ("main" - method or "getPlan()" method. Only needed - if the JAR file does not specify the class - in its manifest. - -m,--jobmanager <host:port> Address of the JobManager (master) to - which to connect. Specify 'yarn-cluster' - as the JobManager to deploy a YARN cluster - for the job. Use this flag to connect to a - different JobManager than the one - specified in the configuration. - -p,--parallelism <parallelism> The parallelism with which to run the - program. Optional flag to override the - default value specified in the - configuration. - Additional arguments if -m yarn-cluster is set: - -yD <arg> Dynamic properties - -yd,--yarndetached Start detached - -yj,--yarnjar <arg> Path to Flink jar file - -yjm,--yarnjobManagerMemory <arg> Memory for JobManager Container [in - MB] - -yn,--yarncontainer <arg> Number of YARN container to allocate - (=Number of Task Managers) - -yq,--yarnquery Display available YARN resources - (memory, cores) - -yqu,--yarnqueue <arg> Specify YARN queue. - -ys,--yarnslots <arg> Number of slots per TaskManager - -yt,--yarnship <arg> Ship files in the specified directory - (t for transfer) - -ytm,--yarntaskManagerMemory <arg> Memory per TaskManager Container [in - MB] - - -Action "info" shows the optimized execution plan of the program (JSON). - - Syntax: info [OPTIONS] <jar-file> <arguments> - "info" action options: - -c,--class <classname> Class with the program entry point ("main" - method or "getPlan()" method. Only needed - if the JAR file does not specify the class - in its manifest. - -m,--jobmanager <host:port> Address of the JobManager (master) to - which to connect. Specify 'yarn-cluster' - as the JobManager to deploy a YARN cluster - for the job. Use this flag to connect to a - different JobManager than the one - specified in the configuration. - -p,--parallelism <parallelism> The parallelism with which to run the - program. Optional flag to override the - default value specified in the - configuration. - - -Action "list" lists running and scheduled programs. - - Syntax: list [OPTIONS] - "list" action options: - -m,--jobmanager <host:port> Address of the JobManager (master) to which - to connect. Specify 'yarn-cluster' as the - JobManager to deploy a YARN cluster for the - job. Use this flag to connect to a different - JobManager than the one specified in the - configuration. - -r,--running Show only running programs and their JobIDs - -s,--scheduled Show only scheduled programs and their JobIDs - - -Action "cancel" cancels a running program. - - Syntax: cancel [OPTIONS] <Job ID> - "cancel" action options: - -m,--jobmanager <host:port> Address of the JobManager (master) to which - to connect. Specify 'yarn-cluster' as the - JobManager to deploy a YARN cluster for the - job. Use this flag to connect to a different - JobManager than the one specified in the - configuration. -</code></pre></div> - - </div> - - <div class="col-sm-10 col-sm-offset-1"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> -</div> - - </div><!-- /.container --> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="http://flink.apache.org/docs/master/page/js/codetabs.js";></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - - <!-- Disqus --> - <script type="text/javascript"> - var disqus_shortname = 'stratosphere-eu'; - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); -</script> - </body> -</html>
![http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"](http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"){kind=link}
![http://flink.apache.org/docs/master/page/img/navbar-brand-logo.jpg"](http://flink.apache.org/docs/master/page/img/navbar-brand-logo.jpg"){kind=link}