http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/master/internals/ide_setup.html ---------------------------------------------------------------------- diff --git a/content/docs/master/internals/ide_setup.html b/content/docs/master/internals/ide_setup.html deleted file mode 100644 index 6b2c1f1..0000000 --- a/content/docs/master/internals/ide_setup.html +++ /dev/null @@ -1,328 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<!DOCTYPE html> - -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - - <title>Apache Flink 0.10-SNAPSHOT Documentation: IDE setup</title> - - <link rel="shortcut icon" href="http://flink.apache.org/docs/master/page/favicon.ico"; type="image/x-icon"> - <link rel="icon" href="http://flink.apache.org/docs/master/page/favicon.ico"; type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/flink.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/syntax.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/codetabs.css";> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - - - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="http://flink.apache.org/docs/master/page/img/navbar-brand-logo.jpg";></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li><a href="http://flink.apache.org/docs/master/index.html";>Overview<span class="hidden-sm hidden-xs"> 0.10</span></a></li> - - <!-- Setup --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/setup"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/setup/building.html";>Get Flink 0.10-SNAPSHOT</a></li> - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - <li><a href="http://flink.apache.org/docs/master/setup/local_setup.html"; class="active">Local</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/cluster_setup.html";>Cluster (Standalone)</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/yarn_setup.html";>YARN</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/gce_setup.html";>GCloud</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/flink_on_tez.html";>Flink on Tez <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="http://flink.apache.org/docs/master/setup/config.html";>Configuration</a></li> - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/apis"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/apis/programming_guide.html";><strong>Batch: DataSet API</strong></a></li> - <li><a href="http://flink.apache.org/docs/master/apis/streaming_guide.html";><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/apis/python.html";>Python API <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="scala_shell.html">Interactive Scala Shell</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/dataset_transformations.html";>Dataset Transformations</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/best_practices.html";>Best Practices</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/example_connectors.html";>Connectors</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/examples.html";>Examples</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/local_execution.html";>Local Execution</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/cluster_execution.html";>Cluster Execution</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/cli.html";>Command Line Interface</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/web_client.html";>Web Client</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/iterations.html";>Iterations</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/java8.html";>Java 8</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/hadoop_compatibility.html";>Hadoop Compatability <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/libs"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/libs/spargel_guide.html";>Graphs: Spargel</a></li> - <li><a href="http://flink.apache.org/docs/master/libs/gelly_guide.html";>Graphs: Gelly <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/libs/ml/";>Machine Learning <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/libs/table.html";>Relational: Table <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Internals --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/internals"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/docs/master/internals/how_to_contribute.html";>How to Contribute</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/coding_guidelines.html";>Coding Guidelines</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/ide_setup.html";>IDE Setup</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/logging.html";>Logging</a></li> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Internals</strong></li> - <li><a href="http://flink.apache.org/docs/master/internals/general_arch.html";>Architecture & Process Model</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/types_serialization.html";>Type Extraction & Serialization</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/job_scheduling.html";>Jobs & Scheduling</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/add_operator.html";>How-To: Add an Operator</a></li> - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/master/search-results.html";> - <div class="form-group"> - <input type="text" class="form-control" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-10 col-sm-offset-1"> - <h1>IDE setup</h1> - - - -<ul id="markdown-toc"> - <li><a href="#eclipse" id="markdown-toc-eclipse">Eclipse</a> <ul> - <li><a href="#eclipse-scala-ide-303" id="markdown-toc-eclipse-scala-ide-303">Eclipse Scala IDE 3.0.3</a></li> - <li><a href="#eclipse-scala-ide-400" id="markdown-toc-eclipse-scala-ide-400">Eclipse Scala IDE 4.0.0</a></li> - </ul> - </li> - <li><a href="#intellij-idea" id="markdown-toc-intellij-idea">IntelliJ IDEA</a> <ul> - <li><a href="#installing-the-scala-plugin" id="markdown-toc-installing-the-scala-plugin">Installing the Scala plugin</a></li> - <li><a href="#installing-the-scala-compiler-plugin" id="markdown-toc-installing-the-scala-compiler-plugin">Installing the Scala compiler plugin</a></li> - <li><a href="#importing-flink" id="markdown-toc-importing-flink">Importing Flink</a></li> - </ul> - </li> -</ul> - -<h2 id="eclipse">Eclipse</h2> - -<p>A brief guide how to set up Eclipse for development of the Flink core. -Flink uses mixed Scala/Java projects, which pose a challenge to some IDEs. -Below is the setup guide that works best from our personal experience.</p> - -<p>For Eclipse users, we currently recomment the Scala IDE 3.0.3, as the most robust solution.</p> - -<h3 id="eclipse-scala-ide-303">Eclipse Scala IDE 3.0.3</h3> - -<p><strong>NOTE:</strong> While this version of the Scala IDE is not the newest, we have found it to be the most reliably working -version for complex projects like Flink. One restriction is, though, that it works only with Java 7, not with Java 8.</p> - -<p><strong>Note:</strong> Before following this setup, make sure to run the build from the command line once -(<code>mvn clean package -DskipTests</code>)</p> - -<ol> - <li>Download the Scala IDE (preferred) or install the plugin to Eclipse Kepler. See section below for download links -and instructions.</li> - <li>Add the âmacroparadiseâ compiler plugin to the Scala compiler. -Open âWindowâ -> âPreferencesâ -> âScalaâ -> âCompilerâ -> âAdvancedâ and put into the âXpluginâ field the path to -the <em>macroparadise</em> jar file (typically â/home/<em>-your-user-</em>/.m2/repository/org/scalamacros/paradise_2.10.4/2.0.1/paradise_2.10.4-2.0.1.jarâ). -Note: If you do not have the jar file, you probably did not ran the command line build.</li> - <li>Import the Flink Maven projects (âFileâ -> âImportâ -> âMavenâ -> âExisting Maven Projectsâ)</li> - <li>During the import, Eclipse will ask to automatically install additional Maven build helper plugins.</li> - <li>Close the âflink-java8â project. Since Eclipse Kepler does not support Java 8, you cannot develop this project.</li> -</ol> - -<h4 id="download-links-for-scala-ide-303">Download links for Scala IDE 3.0.3</h4> - -<p>The Scala IDE 3.0.3 is a previous stable release, and download links are a bit hidden.</p> - -<p>The pre-packaged Scala IDE can be downloaded from the following links:</p> - -<ul> - <li><a href="http://downloads.typesafe.com/scalaide-pack/3.0.3.vfinal-210-20140327/scala-SDK-3.0.3-2.10-linux.gtk.x86_64.tar.gz";>Linux (64 bit)</a></li> - <li><a href="http://downloads.typesafe.com/scalaide-pack/3.0.3.vfinal-210-20140327/scala-SDK-3.0.3-2.10-linux.gtk.x86.tar.gz";>Linux (32 bit)</a></li> - <li><a href="http://downloads.typesafe.com/scalaide-pack/3.0.3.vfinal-210-20140327/scala-SDK-3.0.3-2.10-macosx.cocoa.x86_64.zip";>MaxOS X Cocoa (64 bit)</a></li> - <li><a href="http://downloads.typesafe.com/scalaide-pack/3.0.3.vfinal-210-20140327/scala-SDK-3.0.3-2.10-macosx.cocoa.x86.zip";>MaxOS X Cocoa (32 bit)</a></li> - <li><a href="http://downloads.typesafe.com/scalaide-pack/3.0.3.vfinal-210-20140327/scala-SDK-3.0.3-2.10-win32.win32.x86_64.zip";>Windows (64 bit)</a></li> - <li><a href="http://downloads.typesafe.com/scalaide-pack/3.0.3.vfinal-210-20140327/scala-SDK-3.0.3-2.10-win32.win32.x86.zip";>Windows (32 bit)</a></li> -</ul> - -<p>Alternatively, you can download Eclipse Kepler from <a href="https://eclipse.org/downloads/packages/release/Kepler/SR2";>https://eclipse.org/downloads/packages/release/Kepler/SR2</a> -and manually add the Scala and Maven plugins by plugin site at <a href="http://scala-ide.org/download/prev-stable.html";>http://scala-ide.org/download/prev-stable.html</a>.</p> - -<ul> - <li>Either use the update site to install the plugin (âHelpâ -> âInstall new Softwareâ)</li> - <li>Or download the <a href="http://download.scala-ide.org/sdk/helium/e38/scala211/stable/update-site.zip";>zip file</a>, unpack it, and move the contents of the -âpluginsâ and âfeaturesâ folders into the equally named folders of the Eclipse root directory</li> -</ul> - -<h3 id="eclipse-scala-ide-400">Eclipse Scala IDE 4.0.0</h3> - -<p><strong>NOTE: From personal experience, the use of the Scala IDE 4.0.0 performs worse than previous versions for complex projects like Flink.</strong> -<strong>Version 4.0.0 does not handle mixed Java/Scala projects as robustly and it frequently raises incorrect import and type errors.</strong></p> - -<p><em>Note:</em> Before following this setup, make sure to run the build from the command line once -(<code>mvn clean package -DskipTests</code>)</p> - -<ol> - <li>Download the Scala IDE: <a href="http://scala-ide.org/download/sdk.html";>http://scala-ide.org/download/sdk.html</a></li> - <li>Import the Flink Maven projects (File -> Import -> Maven -> Existing Maven Projects)</li> - <li>While importing the Flink project, the IDE may ask you to install an additional maven build helper plugin.</li> - <li>After the import, you need to set the Scala version of your projects to Scala 2.10 (from the default 2.11). -To do that, select all projects that contain Scala code (marked by the small <em>S</em> on the project icon), -right click and select âScala -> Set the Scala Installationâ and pick â2.10.4â. -Currently, the project to which that is relevant are âflink-runtimeâ, âflink-scalaâ, âflink-scala-examplesâ, -âflink-streaming-exampleâ, âflink-streaming-scalaâ, âflink-testsâ, âflink-test-utilsâ, and âflink-yarnâ.</li> - <li>Depending on your version of the Scala IDE, you may need to add the âmacroparadiseâ compiler plugin to the -Scala compiler. Open âWindowâ -> âPreferencesâ -> âScalaâ -> âCompilerâ -> âAdvancedâ and put into the âXpluginâ field -the path to the <em>macroparadise</em> jar file (typically â/home/<em>-your-user-</em>/.m2/repository/org/scalamacros/paradise_2.10.4/2.0.1/paradise_2.10.4-2.0.1.jarâ)</li> - <li>In order to compile the âflink-java-8â project, you may need to add a Java 8 execution environment. -See <a href="http://stackoverflow.com/questions/25391207/how-do-i-add-execution-environment-1-8-to-eclipse-luna";>this post</a> -for details.</li> -</ol> - -<h2 id="intellij-idea">IntelliJ IDEA</h2> - -<p>A brief guide on how to set up IntelliJ IDEA IDE for development of the Flink core. -As Eclipse is known to have issues with mixed Scala and Java projects, more and more contributers are migrating to IntelliJ IDEA.</p> - -<p>The following documentation describes the steps to setup IntelliJ IDEA 14.0.3 (https://www.jetbrains.com/idea/download/) with the Flink sources.</p> - -<p>Prior to doing anything, make sure that the Flink project is built at least once from the terminal: -<code>mvn clean package -DskipTests</code></p> - -<h3 id="installing-the-scala-plugin">Installing the Scala plugin</h3> -<ol> - <li>Go to IntelliJ plugins settings (File -> Settings -> Plugins) and click on âInstall Jetbrains pluginâ¦â.</li> - <li>Select and install the âScalaâ plugin.</li> - <li>Restart IntelliJ</li> -</ol> - -<h3 id="installing-the-scala-compiler-plugin">Installing the Scala compiler plugin</h3> -<ol> - <li>Go to IntelliJ scala compiler settings (File -> Settings -> Build, Execution, Deployment -> Compiler -> Scala Compiler) and click on âInstall Jetbrains pluginâ¦â.</li> - <li>Click on the green plus icon on the right to add a compiler plugin</li> - <li>Point to the paradise jar: ~/.m2/repository/org/scalamacros/paradise_2.10.4/2.0.1/paradise_2.10.4-2.0.1.jar If there is no such file, this means that you should build Flink from the terminal as explained above.</li> -</ol> - -<h3 id="importing-flink">Importing Flink</h3> -<ol> - <li>Start IntelliJ IDEA and choose âImport Projectâ</li> - <li>Select the root folder of the Flink repository</li> - <li>Choose âImport project from external modelâ and select âMavenâ</li> - <li>Leave the default options and finish the import.</li> -</ol> - - </div> - - <div class="col-sm-10 col-sm-offset-1"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> -</div> - - </div><!-- /.container --> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="http://flink.apache.org/docs/master/page/js/codetabs.js";></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - - <!-- Disqus --> - <script type="text/javascript"> - var disqus_shortname = 'stratosphere-eu'; - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); -</script> - </body> -</html>
http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/master/internals/index.html ---------------------------------------------------------------------- diff --git a/content/docs/master/internals/index.html b/content/docs/master/internals/index.html deleted file mode 100644 index 91a6867..0000000 --- a/content/docs/master/internals/index.html +++ /dev/null @@ -1,205 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<!DOCTYPE html> - -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - - <title>Apache Flink 0.10-SNAPSHOT Documentation: Internals</title> - - <link rel="shortcut icon" href="http://flink.apache.org/docs/master/page/favicon.ico"; type="image/x-icon"> - <link rel="icon" href="http://flink.apache.org/docs/master/page/favicon.ico"; type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/flink.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/syntax.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/codetabs.css";> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - - - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="http://flink.apache.org/docs/master/page/img/navbar-brand-logo.jpg";></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li><a href="http://flink.apache.org/docs/master/index.html";>Overview<span class="hidden-sm hidden-xs"> 0.10</span></a></li> - - <!-- Setup --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/setup"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/setup/building.html";>Get Flink 0.10-SNAPSHOT</a></li> - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - <li><a href="http://flink.apache.org/docs/master/setup/local_setup.html"; class="active">Local</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/cluster_setup.html";>Cluster (Standalone)</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/yarn_setup.html";>YARN</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/gce_setup.html";>GCloud</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/flink_on_tez.html";>Flink on Tez <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="http://flink.apache.org/docs/master/setup/config.html";>Configuration</a></li> - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/apis"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/apis/programming_guide.html";><strong>Batch: DataSet API</strong></a></li> - <li><a href="http://flink.apache.org/docs/master/apis/streaming_guide.html";><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/apis/python.html";>Python API <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="scala_shell.html">Interactive Scala Shell</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/dataset_transformations.html";>Dataset Transformations</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/best_practices.html";>Best Practices</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/example_connectors.html";>Connectors</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/examples.html";>Examples</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/local_execution.html";>Local Execution</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/cluster_execution.html";>Cluster Execution</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/cli.html";>Command Line Interface</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/web_client.html";>Web Client</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/iterations.html";>Iterations</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/java8.html";>Java 8</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/hadoop_compatibility.html";>Hadoop Compatability <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/libs"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/libs/spargel_guide.html";>Graphs: Spargel</a></li> - <li><a href="http://flink.apache.org/docs/master/libs/gelly_guide.html";>Graphs: Gelly <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/libs/ml/";>Machine Learning <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/libs/table.html";>Relational: Table <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Internals --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/internals"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/docs/master/internals/how_to_contribute.html";>How to Contribute</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/coding_guidelines.html";>Coding Guidelines</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/ide_setup.html";>IDE Setup</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/logging.html";>Logging</a></li> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Internals</strong></li> - <li><a href="http://flink.apache.org/docs/master/internals/general_arch.html";>Architecture & Process Model</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/types_serialization.html";>Type Extraction & Serialization</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/job_scheduling.html";>Jobs & Scheduling</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/add_operator.html";>How-To: Add an Operator</a></li> - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/master/search-results.html";> - <div class="form-group"> - <input type="text" class="form-control" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-10 col-sm-offset-1"> - <h1>Internals</h1> - - - - </div> - - <div class="col-sm-10 col-sm-offset-1"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> -</div> - - </div><!-- /.container --> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="http://flink.apache.org/docs/master/page/js/codetabs.js";></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - - <!-- Disqus --> - <script type="text/javascript"> - var disqus_shortname = 'stratosphere-eu'; - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); -</script> - </body> -</html> http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/master/internals/job_scheduling.html ---------------------------------------------------------------------- diff --git a/content/docs/master/internals/job_scheduling.html b/content/docs/master/internals/job_scheduling.html deleted file mode 100644 index a6d13ce..0000000 --- a/content/docs/master/internals/job_scheduling.html +++ /dev/null @@ -1,265 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<!DOCTYPE html> - -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - - <title>Apache Flink 0.10-SNAPSHOT Documentation: Jobs and Scheduling</title> - - <link rel="shortcut icon" href="http://flink.apache.org/docs/master/page/favicon.ico"; type="image/x-icon"> - <link rel="icon" href="http://flink.apache.org/docs/master/page/favicon.ico"; type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/flink.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/syntax.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/codetabs.css";> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - - - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="http://flink.apache.org/docs/master/page/img/navbar-brand-logo.jpg";></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li><a href="http://flink.apache.org/docs/master/index.html";>Overview<span class="hidden-sm hidden-xs"> 0.10</span></a></li> - - <!-- Setup --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/setup"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/setup/building.html";>Get Flink 0.10-SNAPSHOT</a></li> - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - <li><a href="http://flink.apache.org/docs/master/setup/local_setup.html"; class="active">Local</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/cluster_setup.html";>Cluster (Standalone)</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/yarn_setup.html";>YARN</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/gce_setup.html";>GCloud</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/flink_on_tez.html";>Flink on Tez <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="http://flink.apache.org/docs/master/setup/config.html";>Configuration</a></li> - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/apis"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/apis/programming_guide.html";><strong>Batch: DataSet API</strong></a></li> - <li><a href="http://flink.apache.org/docs/master/apis/streaming_guide.html";><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/apis/python.html";>Python API <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="scala_shell.html">Interactive Scala Shell</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/dataset_transformations.html";>Dataset Transformations</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/best_practices.html";>Best Practices</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/example_connectors.html";>Connectors</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/examples.html";>Examples</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/local_execution.html";>Local Execution</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/cluster_execution.html";>Cluster Execution</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/cli.html";>Command Line Interface</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/web_client.html";>Web Client</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/iterations.html";>Iterations</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/java8.html";>Java 8</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/hadoop_compatibility.html";>Hadoop Compatability <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/libs"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/libs/spargel_guide.html";>Graphs: Spargel</a></li> - <li><a href="http://flink.apache.org/docs/master/libs/gelly_guide.html";>Graphs: Gelly <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/libs/ml/";>Machine Learning <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/libs/table.html";>Relational: Table <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Internals --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/internals"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/docs/master/internals/how_to_contribute.html";>How to Contribute</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/coding_guidelines.html";>Coding Guidelines</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/ide_setup.html";>IDE Setup</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/logging.html";>Logging</a></li> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Internals</strong></li> - <li><a href="http://flink.apache.org/docs/master/internals/general_arch.html";>Architecture & Process Model</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/types_serialization.html";>Type Extraction & Serialization</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/job_scheduling.html";>Jobs & Scheduling</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/add_operator.html";>How-To: Add an Operator</a></li> - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/master/search-results.html";> - <div class="form-group"> - <input type="text" class="form-control" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-10 col-sm-offset-1"> - <h1>Jobs and Scheduling</h1> - - - -<p>This document briefly describes how Flink schedules jobs and -how it represents and tracks job status on the JobManager.</p> - -<ul id="markdown-toc"> - <li><a href="#scheduling" id="markdown-toc-scheduling">Scheduling</a></li> - <li><a href="#jobmanager-data-structures" id="markdown-toc-jobmanager-data-structures">JobManager Data Structures</a></li> -</ul> - -<h2 id="scheduling">Scheduling</h2> - -<p>Execution resources in Flink are defined through <em>Task Slots</em>. Each TaskManager will have one or more task slots, -each of which can run one pipeline of parallel tasks. A pipeline consists of multiple successive tasks, such as the -<em>n-th</em> parallel instance of a MapFunction together with the <em>n-th</em> parallel instance of a ReduceFunction. -Note that Flink often executes successive tasks concurrently: For Streaming programs, that happens in any case, -but also for batch programs, it happens frequently.</p> - -<p>The figure below illustrates that. Consider a program with a data source, a <em>MapFunction</em>, and a <em>ReduceFunction</em>. -The source and MapFunction are executed with a parallelism of 4, while the ReduceFunction is executed with a -parallism of 3. A pipeline consists of the sequence Source - Map - Reduce. On a cluster with 2 TaskManagers with -3 slots each, the program will be executed as described below.</p> - -<div style="text-align: center;"> -<img src="fig/slots.svg" alt="Assigning Pipelines of Tasks to Slots" height="250px" style="text-align: center;" /> -</div> - -<p>Internally, Flink defines through <a href="https://github.com/apache/flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/SlotSharingGroup.java";>SlotSharingGroup</a> -and <a href="https://github.com/apache/flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/CoLocationGroup.java";>CoLocationGroup</a> -which tasks may share a slot (permissive), respectively which tasks must be strictly placed into the same slot.</p> - -<h2 id="jobmanager-data-structures">JobManager Data Structures</h2> - -<p>During job execution, the JobManager keeps track of distributed tasks, decides when to schedule the next task (or set of tasks), -and reacts to finished tasks or execution failures.</p> - -<p>The JobManager receives the <a href="https://github.com/apache/flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/";>JobGraph</a>, -which is a representation of the data flow consisting of operators (<a href="https://github.com/apache/flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/AbstractJobVertex.java";>JobVertex</a>) -and intermediate results (<a href="https://github.com/apache/flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateDataSet.java";>IntermediateDataSet</a>). -Each operator has properies, like the parallelism and the code that it executes. -In addition, the JobGraph has a set of attached libraries, that are neccessary to execute the code of the operators.</p> - -<p>The JobManager transforms the JobGraph into an <a href="https://github.com/apache/flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/";>ExecutionGraph</a>. -The ExecutionGraph is a parallel version of the JobGraph: For each JobVertex, it contains an <a href="https://github.com/apache/flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java";>ExecutionVertex</a> per parallel subtask. An operator with a parallelism of 100 will have one JobVertex and 100 ExecutionVertices. -The ExecutionVertex tracks the state of execution of a particular subtask. All ExecutionVertices from one JobVertex are held in an -<a href="https://github.com/apache/flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java";>ExecutionJobVertex</a>, -which tracks the status of the operator as a whole. -Besides the vertices, the ExecutionGraph also contains the <a href="https://github.com/apache/flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IntermediateResult.java";>IntermediateResult</a> and the <a href="https://github.com/apache/flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IntermediateResultPartition.java";>IntermediateResultPartition</a>. The former tracks the state of the <em>IntermediateDataSet</em>, the latter the state of each of its partitions.</p> - -<div style="text-align: center;"> -<img src="fig/job_and_execution_graph.svg" alt="JobGraph and ExecutionGraph" height="400px" style="text-align: center;" /> -</div> - -<p>During its execution, each parallel task goes through multiple stages, from <em>created</em> to <em>finished</em> or <em>failed</em>. The diagram below illustrates the -states and possible transitions between them. A task may be executed multiple times (for example in the course of failure recovery). -For that reason, the execution of an ExecutionVertex is tracked in an <a href="https://github.com/apache/flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java";>Execution</a>. Each ExecutionVertex has a current Execution, and prior Executions.</p> - -<div style="text-align: center;"> -<img src="fig/state_machine.svg" alt="States and Transitions of Task Executions" height="300px" style="text-align: center;" /> -</div> - - - </div> - - <div class="col-sm-10 col-sm-offset-1"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> -</div> - - </div><!-- /.container --> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="http://flink.apache.org/docs/master/page/js/codetabs.js";></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - - <!-- Disqus --> - <script type="text/javascript"> - var disqus_shortname = 'stratosphere-eu'; - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); -</script> - </body> -</html> http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/master/internals/logging.html ---------------------------------------------------------------------- diff --git a/content/docs/master/internals/logging.html b/content/docs/master/internals/logging.html deleted file mode 100644 index c9f17fb..0000000 --- a/content/docs/master/internals/logging.html +++ /dev/null @@ -1,273 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<!DOCTYPE html> - -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - - <title>Apache Flink 0.10-SNAPSHOT Documentation: How to use logging</title> - - <link rel="shortcut icon" href="http://flink.apache.org/docs/master/page/favicon.ico"; type="image/x-icon"> - <link rel="icon" href="http://flink.apache.org/docs/master/page/favicon.ico"; type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/flink.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/syntax.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/master/page/css/codetabs.css";> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - - - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="http://flink.apache.org/docs/master/page/img/navbar-brand-logo.jpg";></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li><a href="http://flink.apache.org/docs/master/index.html";>Overview<span class="hidden-sm hidden-xs"> 0.10</span></a></li> - - <!-- Setup --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/setup"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/setup/building.html";>Get Flink 0.10-SNAPSHOT</a></li> - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - <li><a href="http://flink.apache.org/docs/master/setup/local_setup.html"; class="active">Local</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/cluster_setup.html";>Cluster (Standalone)</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/yarn_setup.html";>YARN</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/gce_setup.html";>GCloud</a></li> - <li><a href="http://flink.apache.org/docs/master/setup/flink_on_tez.html";>Flink on Tez <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="http://flink.apache.org/docs/master/setup/config.html";>Configuration</a></li> - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/apis"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/apis/programming_guide.html";><strong>Batch: DataSet API</strong></a></li> - <li><a href="http://flink.apache.org/docs/master/apis/streaming_guide.html";><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/apis/python.html";>Python API <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="scala_shell.html">Interactive Scala Shell</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/dataset_transformations.html";>Dataset Transformations</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/best_practices.html";>Best Practices</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/example_connectors.html";>Connectors</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/examples.html";>Examples</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/local_execution.html";>Local Execution</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/cluster_execution.html";>Cluster Execution</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/cli.html";>Command Line Interface</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/web_client.html";>Web Client</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/iterations.html";>Iterations</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/java8.html";>Java 8</a></li> - <li><a href="http://flink.apache.org/docs/master/apis/hadoop_compatibility.html";>Hadoop Compatability <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/libs"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/master/libs/spargel_guide.html";>Graphs: Spargel</a></li> - <li><a href="http://flink.apache.org/docs/master/libs/gelly_guide.html";>Graphs: Gelly <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/libs/ml/";>Machine Learning <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/master/libs/table.html";>Relational: Table <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Internals --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/master/internals"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/docs/master/internals/how_to_contribute.html";>How to Contribute</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/coding_guidelines.html";>Coding Guidelines</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/ide_setup.html";>IDE Setup</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/logging.html";>Logging</a></li> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Internals</strong></li> - <li><a href="http://flink.apache.org/docs/master/internals/general_arch.html";>Architecture & Process Model</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/types_serialization.html";>Type Extraction & Serialization</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/job_scheduling.html";>Jobs & Scheduling</a></li> - <li><a href="http://flink.apache.org/docs/master/internals/add_operator.html";>How-To: Add an Operator</a></li> - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/master/search-results.html";> - <div class="form-group"> - <input type="text" class="form-control" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-10 col-sm-offset-1"> - <h1>How to use logging</h1> - - - -<p>The logging in Flink is implemented using the slf4j logging interface. As underlying logging framework, log4j is used. We also provide logback configuration files and pass them to the JVMâs as properties. Users willing to use logback instead of log4j can just exclude log4j (or delete it from the lib/ folder).</p> - -<ul id="markdown-toc"> - <li><a href="#configuring-log4j" id="markdown-toc-configuring-log4j">Configuring Log4j</a></li> - <li><a href="#configuring-logback" id="markdown-toc-configuring-logback">Configuring logback</a></li> - <li><a href="#best-practices-for-developers" id="markdown-toc-best-practices-for-developers">Best practices for developers</a></li> -</ul> - -<h2 id="configuring-log4j">Configuring Log4j</h2> - -<p>Log4j is controlled using property files. In Flinkâs case, the file is usually called <code>log4j.properties</code>. We pass the filename and location of this file using the <code>-Dlog4j.configuration=</code> parameter to the JVM.</p> - -<h2 id="configuring-logback">Configuring logback</h2> - -<p>For users and developers alike it is important to control the logging framework. -The configuration of the logging framework is exclusively done by configuration files. -The configuration file either has to be specified by setting the environment property <code>-Dlogback.configurationFile=<file></code> or by putting <code>logback.xml</code> in the classpath. -The <code>conf</code> directory contains a <code>logback.xml</code> file which can be modified and is used if Flink is started outside of an IDE and with the provided starting scripts. -The provided <code>logback.xml</code> has the following form:</p> - -<div class="highlight"><pre><code class="language-xml"><span class="nt"><configuration></span> - <span class="nt"><appender</span> <span class="na">name=</span><span class="s">"file"</span> <span class="na">class=</span><span class="s">"ch.qos.logback.core.FileAppender"</span><span class="nt">></span> - <span class="nt"><file></span>${log.file}<span class="nt"></file></span> - <span class="nt"><append></span>false<span class="nt"></append></span> - <span class="nt"><encoder></span> - <span class="nt"><pattern></span>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n<span class="nt"></pattern></span> - <span class="nt"></encoder></span> - <span class="nt"></appender></span> - - <span class="nt"><root</span> <span class="na">level=</span><span class="s">"INFO"</span><span class="nt">></span> - <span class="nt"><appender-ref</span> <span class="na">ref=</span><span class="s">"file"</span><span class="nt">/></span> - <span class="nt"></root></span> -<span class="nt"></configuration></span></code></pre></div> - -<p>In order to control the logging level of <code>org.apache.flink.runtime.jobgraph.JobGraph</code>, for example, one would have to add the following line to the configuration file.</p> - -<div class="highlight"><pre><code class="language-xml"><span class="nt"><logger</span> <span class="na">name=</span><span class="s">"org.apache.flink.runtime.jobgraph.JobGraph"</span> <span class="na">level=</span><span class="s">"DEBUG"</span><span class="nt">/></span></code></pre></div> - -<p>For further information on configuring logback see <a href="http://logback.qos.ch/manual/configuration.html";>LOGbackâs manual</a>.</p> - -<h2 id="best-practices-for-developers">Best practices for developers</h2> - -<p>The loggers using slf4j are created by calling</p> - -<div class="highlight"><pre><code class="language-java"><span class="kn">import</span> <span class="nn">org.slf4j.LoggerFactory</span> -<span class="kn">import</span> <span class="nn">org.slf4j.Logger</span> - -<span class="n">Logger</span> <span class="n">LOG</span> <span class="o">=</span> <span class="n">LoggerFactory</span><span class="o">.</span><span class="na">getLogger</span><span class="o">(</span><span class="n">Foobar</span><span class="o">.</span><span class="na">class</span><span class="o">)</span></code></pre></div> - -<p>In order to benefit most from slf4j, it is recommended to use its placeholder mechanism. -Using placeholders allows to avoid unnecessary string constructions in case that the logging level is set so high that the message would not be logged. -The syntax of placeholders is the following:</p> - -<div class="highlight"><pre><code class="language-java"><span class="n">LOG</span><span class="o">.</span><span class="na">info</span><span class="o">(</span><span class="s">"This message contains {} placeholders. {}"</span><span class="o">,</span> <span class="mi">2</span><span class="o">,</span> <span class="s">"Yippie"</span><span class="o">);</span></code></pre></div> - -<p>Placeholders can also be used in conjunction with exceptions which shall be logged.</p> - -<div class="highlight"><pre><code class="language-java"><span class="k">catch</span><span class="o">(</span><span class="n">Exception</span> <span class="n">exception</span><span class="o">){</span> - <span class="n">LOG</span><span class="o">.</span><span class="na">error</span><span class="o">(</span><span class="s">"An {} occurred."</span><span class="o">,</span> <span class="s">"error"</span><span class="o">,</span> <span class="n">exception</span><span class="o">);</span> -<span class="o">}</span></code></pre></div> - -<hr /> - -<p><em>This documentation is maintained by the contributors of the individual components. -We kindly ask anyone that adds and changes components to eventually provide a patch -or pull request that updates these documents as well.</em></p> - - - </div> - - <div class="col-sm-10 col-sm-offset-1"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> -</div> - - </div><!-- /.container --> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="http://flink.apache.org/docs/master/page/js/codetabs.js";></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - - <!-- Disqus --> - <script type="text/javascript"> - var disqus_shortname = 'stratosphere-eu'; - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); -</script> - </body> -</html>
![http://flink.apache.org/docs/master/page/img/navbar-brand-logo.jpg"](http://flink.apache.org/docs/master/page/img/navbar-brand-logo.jpg"){kind=link}