Modified: flink/site/docs/0.6-incubating/how_to_contribute.html URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/how_to_contribute.html?rev=1658387&r1=1658386&r2=1658387&view=diff ============================================================================== --- flink/site/docs/0.6-incubating/how_to_contribute.html (original) +++ flink/site/docs/0.6-incubating/how_to_contribute.html Mon Feb 9 12:40:44 2015 @@ -5,135 +5,112 @@ <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1"> <title>Apache Flink (incubating): How to contribute</title> - <link rel="shortcut icon" href="favicon.ico" type="image/x-icon"> - <link rel="icon" href="favicon.ico" type="image/x-icon"> - <link rel="stylesheet" href="css/bootstrap.css"> - <link rel="stylesheet" href="css/bootstrap-lumen-custom.css"> - <link rel="stylesheet" href="css/syntax.css"> - <link rel="stylesheet" href="css/custom.css"> - <link href="css/main/main.css" rel="stylesheet"> + <link rel="stylesheet" href="/css/bootstrap.css"> + <link rel="stylesheet" href="/css/bootstrap-lumen-custom.css"> + <link rel="stylesheet" href="/css/syntax.css"> + <link rel="/css/custom.css"> + <link rel="css/codetabs.css"> + <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.1.0/css/font-awesome.min.css" rel="stylesheet"> <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js";></script> - <script src="js/bootstrap.min.js"></script> + <script src="/js/bootstrap.min.js"></script> <script src="js/codetabs.js"></script> </head> <body> <nav class="navbar navbar-default navbar-fixed-top" role="navigation"> <div class="container"> - <div class="row"> - <div class="col-md-1 af-mobile-nav-bar"> - <a href="index.html" title="Home"> - <img class="hidden-xs hidden-sm img-responsive" - src="img/logo.png" alt="Apache Flink Logo"> - </a> - <div class="row visible-xs"> - <div class="col-xs-3"> - <a href="index.html" title="Home"> - <img class="hidden-x hidden-sm img-responsive" - src="img/logo.png" alt="Apache Flink Logo"> - </a> - </div> - <div class="col-xs-5"></div> - <div class="col-xs-4"> - <div class="af-mobile-btn"> - <span class="glyphicon glyphicon-plus"></span> - </div> - </div> - </div> - </div> - <!-- Navigation --> - <div class="col-md-11"> - <div class="collapse navbar-collapse" id="navbar-collapse-1"> - <ul class="nav navbar-nav"> - - <li> - <a href="index.html" class="">Documentation</a> - </li> - - <li> - <a href="api/java/index.html">Javadoc</a> - </li> - - <li> - <a href="api/scala/index.html#org.apache.flink.api.scala.package">Scaladoc</a> - </li> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="http://flink.incubator.apache.org";>Apache Flink</a> + </div> - </ul> - </div> - </div> + <div class="collapse navbar-collapse" id="navbar-collapse-1"> + <ul class="nav navbar-nav"> + + <li> + <a href="index.html" class="">Documentation</a> + </li> + + <li> + <a href="api/java/index.html">Javadoc</a> + </li> + </ul> </div> </div> </nav> - - <div style="padding-top:120px" class="container"> + <div style="padding-top:70px" class="container"> <div class="row"> <div class="col-md-3"> <ul> - <li><a href="faq.html">FAQ</a></li> - <li>Quickstart - <ul> - <li><a href="setup_quickstart.html">Setup</a></li> - <li><a href="run_example_quickstart.html">Run Example</a></li> - <li><a href="java_api_quickstart.html">Java API</a></li> - <li><a href="scala_api_quickstart.html">Scala API</a></li> - </ul> - </li> - - <li>Setup & Configuration - <ul> - <li><a href="local_setup.html">Local Setup</a></li> - <li><a href="building.html">Build Flink</a></li> - <li><a href="cluster_setup.html">Cluster Setup</a></li> - <li><a href="yarn_setup.html">YARN Setup</a></li> - <li><a href="config.html">Configuration</a></li> - </ul> - </li> - - <li>Programming Guides - <ul> - <li><a href="programming_guide.html">Programming Guide</a></li> - <li><a href="dataset_transformations.html">DataSet Transformations</a></li> - <li><a href="java8_programming_guide.html">Java 8 Programming Guide</a></li> - <li><a href="streaming_guide.html">Streaming Guide</a></li> - <li><a href="iterations.html">Iterations</a></li> - <li><a href="spargel_guide.html">Spargel Graph API</a></li> - <li><a href="hadoop_compatibility.html">Hadoop Compatibility</a></li> - </ul> - </li> - - <li>Examples - <ul> - <li><a href="examples.html">Bundled Examples</a></li> - <li><a href="example_connectors.html">Connecting to other systems</a></li> - </ul> - </li> - - <li>Execution - <ul> - <li><a href="local_execution.html">Local/Debugging</a></li> - <li><a href="cluster_execution.html">Cluster</a></li> - <li><a href="cli.html">Command-Line Interface</a></li> - <li><a href="web_client.html">Web Interface</a></li> - </ul> - </li> - - <li>Internals - <ul> - <li><a href="internal_overview.html">Overview</a></li> - </ul> - </li> + <li>Quickstart + <ul> + <li><a href="setup_quickstart.html">Install</a></li> + <li><a href="run_example_quickstart.html">Run Example</a></li> + <li><a href="java_api_quickstart.html">Java API</a></li> + <li><a href="scala_api_quickstart.html">Scala API</a></li> + <li><a href="faq.html">FAQ</a></li> + </ul> + </li> + + <li>Setup & Configuration + <ul> + <li><a href="building.html">Build Instructions</a></li> + <li><a href="local_setup.html">Local Setup</a></li> + <li><a href="cluster_setup.html">Cluster Setup</a></li> + <li><a href="yarn_setup.html">YARN Setup</a></li> + <li><a href="config.html">Configuration</a></li> + </ul> + </li> + + <li>Programming Guides + <ul> + <li><a href="java_api_guide.html">Java API</a></li> + <li><a href="java_api_transformations.html">Java API Transformations</a></li> + <li><a href="scala_api_guide.html">Scala API</a></li> + <li><a href="iterations.html">Iterations</a></li> + <li><a href="spargel_guide.html">Spargel Graph API</a></li> + </ul> + </li> + + <li>Examples + <ul> + <li><a href="java_api_examples.html">Java API</a></li> + <li><a href="scala_api_examples.html">Scala API</a></li> + <li><a href="example_connectors.html">Connecting to other systems</a></li> + </ul> + </li> + + <li>Execution + <ul> + <li><a href="local_execution.html">Local/Debugging</a></li> + <li><a href="cluster_execution.html">Cluster</a></li> + <li><a href="cli.html">Command-Line Interface</a></li> + <li><a href="web_client.html">Web Interface</a></li> + </ul> + </li> + + <li>Internals + <ul> + <li><a href="internal_overview.html">Overview</a></li> + <li><a href="internal_general_arch.html">General Architecture</a></li> + <li><a href="internal_add_operator.html">How-to: Adding a new Operator</a></li> + </ul> + </li> </ul> - </div> <div class="col-md-9"> <h1>How to contribute</h1> - - <p>The âHow to contributeâ-guide is now located <a href="http://flink.incubator.apache.org/how-to-contribute.html";>on the project website</a>.</p> - - <!-- Disqus Area --> + <p>The "How to contribute"-guide is now located <a href="http://flink.incubator.apache.org/how-to-contribute.html";>on the project website</a>.</p> + + <div style="padding-top:30px" id="disqus_thread"></div> <script type="text/javascript"> @@ -148,12 +125,14 @@ })(); </script> <noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript";>comments powered by Disqus.</a></noscript> + <a href="http://disqus.com"; class="dsq-brlink">comments powered by <span class="logo-disqus">Disqus</span></a> + </div> </div> <div class="footer"> - <hr class="divider" /> + <p><hr class="divider"></p> <p><small>Apache Flink is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC. Incubation is @@ -164,10 +143,9 @@ incubation status is not necessarily a r stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.</small></p> -<p><a href="http://incubator.apache.org/";><img src="/img/apache-incubator-logo.png" alt="Incubator Logo" /></a></p> +<p><a href="http://incubator.apache.org/";><img src="/img/apache-incubator-logo.png" alt="Incubator Logo"></a></p> -<p class="text-center"><a href="privacy-policy.html">Privacy Policy<a> -</a></a></p> +<p class="text-center"><a href="/privacy-policy.html">Privacy Policy<a></p> </div> </div>
Modified: flink/site/docs/0.6-incubating/img/FlinkOnYarn.svg URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/img/FlinkOnYarn.svg?rev=1658387&r1=1658386&r2=1658387&view=diff ============================================================================== Binary files - no diff available. Modified: flink/site/docs/0.6-incubating/img/quickstart-example/compiler-webclient-new.png URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/img/quickstart-example/compiler-webclient-new.png?rev=1658387&r1=1658386&r2=1658387&view=diff ============================================================================== Binary files - no diff available. Modified: flink/site/docs/0.6-incubating/img/quickstart-example/jobmanager-running-new.png URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/img/quickstart-example/jobmanager-running-new.png?rev=1658387&r1=1658386&r2=1658387&view=diff ============================================================================== Binary files - no diff available. Modified: flink/site/docs/0.6-incubating/img/quickstart-example/kmeans003.png URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/img/quickstart-example/kmeans003.png?rev=1658387&r1=1658386&r2=1658387&view=diff ============================================================================== Binary files - no diff available. Modified: flink/site/docs/0.6-incubating/img/quickstart-example/kmeans008.png URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/img/quickstart-example/kmeans008.png?rev=1658387&r1=1658386&r2=1658387&view=diff ============================================================================== Binary files - no diff available. Modified: flink/site/docs/0.6-incubating/img/quickstart-example/kmeans015.png URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/img/quickstart-example/kmeans015.png?rev=1658387&r1=1658386&r2=1658387&view=diff ============================================================================== Binary files - no diff available. Modified: flink/site/docs/0.6-incubating/img/quickstart-example/result003.png URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/img/quickstart-example/result003.png?rev=1658387&r1=1658386&r2=1658387&view=diff ============================================================================== Binary files - no diff available. Modified: flink/site/docs/0.6-incubating/img/quickstart-example/result008.png URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/img/quickstart-example/result008.png?rev=1658387&r1=1658386&r2=1658387&view=diff ============================================================================== Binary files - no diff available. Modified: flink/site/docs/0.6-incubating/img/quickstart-example/result015.png URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/img/quickstart-example/result015.png?rev=1658387&r1=1658386&r2=1658387&view=diff ============================================================================== Binary files - no diff available. Modified: flink/site/docs/0.6-incubating/img/quickstart-example/run-webclient.png URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/img/quickstart-example/run-webclient.png?rev=1658387&r1=1658386&r2=1658387&view=diff ============================================================================== Binary files - no diff available. Modified: flink/site/docs/0.6-incubating/index.html URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/index.html?rev=1658387&r1=1658386&r2=1658387&view=diff ============================================================================== --- flink/site/docs/0.6-incubating/index.html (original) +++ flink/site/docs/0.6-incubating/index.html Mon Feb 9 12:40:44 2015 @@ -5,153 +5,131 @@ <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1"> <title>Apache Flink (incubating): Flink Overview</title> - <link rel="shortcut icon" href="favicon.ico" type="image/x-icon"> - <link rel="icon" href="favicon.ico" type="image/x-icon"> - <link rel="stylesheet" href="css/bootstrap.css"> - <link rel="stylesheet" href="css/bootstrap-lumen-custom.css"> - <link rel="stylesheet" href="css/syntax.css"> - <link rel="stylesheet" href="css/custom.css"> - <link href="css/main/main.css" rel="stylesheet"> + <link rel="stylesheet" href="/css/bootstrap.css"> + <link rel="stylesheet" href="/css/bootstrap-lumen-custom.css"> + <link rel="stylesheet" href="/css/syntax.css"> + <link rel="/css/custom.css"> + <link rel="css/codetabs.css"> + <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.1.0/css/font-awesome.min.css" rel="stylesheet"> <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js";></script> - <script src="js/bootstrap.min.js"></script> + <script src="/js/bootstrap.min.js"></script> <script src="js/codetabs.js"></script> </head> <body> <nav class="navbar navbar-default navbar-fixed-top" role="navigation"> <div class="container"> - <div class="row"> - <div class="col-md-1 af-mobile-nav-bar"> - <a href="index.html" title="Home"> - <img class="hidden-xs hidden-sm img-responsive" - src="img/logo.png" alt="Apache Flink Logo"> - </a> - <div class="row visible-xs"> - <div class="col-xs-3"> - <a href="index.html" title="Home"> - <img class="hidden-x hidden-sm img-responsive" - src="img/logo.png" alt="Apache Flink Logo"> - </a> - </div> - <div class="col-xs-5"></div> - <div class="col-xs-4"> - <div class="af-mobile-btn"> - <span class="glyphicon glyphicon-plus"></span> - </div> - </div> - </div> - </div> - <!-- Navigation --> - <div class="col-md-11"> - <div class="collapse navbar-collapse" id="navbar-collapse-1"> - <ul class="nav navbar-nav"> - - <li> - <a href="index.html" class="active">Documentation</a> - </li> - - <li> - <a href="api/java/index.html">Javadoc</a> - </li> - - <li> - <a href="api/scala/index.html#org.apache.flink.api.scala.package">Scaladoc</a> - </li> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="http://flink.incubator.apache.org";>Apache Flink</a> + </div> - </ul> - </div> - </div> + <div class="collapse navbar-collapse" id="navbar-collapse-1"> + <ul class="nav navbar-nav"> + + <li> + <a href="index.html" class="active">Documentation</a> + </li> + + <li> + <a href="api/java/index.html">Javadoc</a> + </li> + </ul> </div> </div> </nav> - - <div style="padding-top:120px" class="container"> + <div style="padding-top:70px" class="container"> <div class="row"> <div class="col-md-3"> <ul> - <li><a href="faq.html">FAQ</a></li> - <li>Quickstart - <ul> - <li><a href="setup_quickstart.html">Setup</a></li> - <li><a href="run_example_quickstart.html">Run Example</a></li> - <li><a href="java_api_quickstart.html">Java API</a></li> - <li><a href="scala_api_quickstart.html">Scala API</a></li> - </ul> - </li> - - <li>Setup & Configuration - <ul> - <li><a href="local_setup.html">Local Setup</a></li> - <li><a href="building.html">Build Flink</a></li> - <li><a href="cluster_setup.html">Cluster Setup</a></li> - <li><a href="yarn_setup.html">YARN Setup</a></li> - <li><a href="config.html">Configuration</a></li> - </ul> - </li> - - <li>Programming Guides - <ul> - <li><a href="programming_guide.html">Programming Guide</a></li> - <li><a href="dataset_transformations.html">DataSet Transformations</a></li> - <li><a href="java8_programming_guide.html">Java 8 Programming Guide</a></li> - <li><a href="streaming_guide.html">Streaming Guide</a></li> - <li><a href="iterations.html">Iterations</a></li> - <li><a href="spargel_guide.html">Spargel Graph API</a></li> - <li><a href="hadoop_compatibility.html">Hadoop Compatibility</a></li> - </ul> - </li> - - <li>Examples - <ul> - <li><a href="examples.html">Bundled Examples</a></li> - <li><a href="example_connectors.html">Connecting to other systems</a></li> - </ul> - </li> - - <li>Execution - <ul> - <li><a href="local_execution.html">Local/Debugging</a></li> - <li><a href="cluster_execution.html">Cluster</a></li> - <li><a href="cli.html">Command-Line Interface</a></li> - <li><a href="web_client.html">Web Interface</a></li> - </ul> - </li> - - <li>Internals - <ul> - <li><a href="internal_overview.html">Overview</a></li> - </ul> - </li> + <li>Quickstart + <ul> + <li><a href="setup_quickstart.html">Install</a></li> + <li><a href="run_example_quickstart.html">Run Example</a></li> + <li><a href="java_api_quickstart.html">Java API</a></li> + <li><a href="scala_api_quickstart.html">Scala API</a></li> + <li><a href="faq.html">FAQ</a></li> + </ul> + </li> + + <li>Setup & Configuration + <ul> + <li><a href="building.html">Build Instructions</a></li> + <li><a href="local_setup.html">Local Setup</a></li> + <li><a href="cluster_setup.html">Cluster Setup</a></li> + <li><a href="yarn_setup.html">YARN Setup</a></li> + <li><a href="config.html">Configuration</a></li> + </ul> + </li> + + <li>Programming Guides + <ul> + <li><a href="java_api_guide.html">Java API</a></li> + <li><a href="java_api_transformations.html">Java API Transformations</a></li> + <li><a href="scala_api_guide.html">Scala API</a></li> + <li><a href="iterations.html">Iterations</a></li> + <li><a href="spargel_guide.html">Spargel Graph API</a></li> + </ul> + </li> + + <li>Examples + <ul> + <li><a href="java_api_examples.html">Java API</a></li> + <li><a href="scala_api_examples.html">Scala API</a></li> + <li><a href="example_connectors.html">Connecting to other systems</a></li> + </ul> + </li> + + <li>Execution + <ul> + <li><a href="local_execution.html">Local/Debugging</a></li> + <li><a href="cluster_execution.html">Cluster</a></li> + <li><a href="cli.html">Command-Line Interface</a></li> + <li><a href="web_client.html">Web Interface</a></li> + </ul> + </li> + + <li>Internals + <ul> + <li><a href="internal_overview.html">Overview</a></li> + <li><a href="internal_general_arch.html">General Architecture</a></li> + <li><a href="internal_add_operator.html">How-to: Adding a new Operator</a></li> + </ul> + </li> </ul> - </div> <div class="col-md-9"> <h1>Flink Overview</h1> - + <p>Apache Flink (incubating) is a platform for efficient, distributed, general-purpose data processing. It features powerful programming abstractions in Java and Scala, a high-performance runtime, and automatic program optimization. It has native support for iterations, incremental iterations, and programs consisting of large DAGs of operations.</p> <p>If you quickly want to try out the system, please look at one of the available quickstarts. For -a thorough introduction of the Flink API please refer to the -<a href="programming_guide.html">Programming Guide</a>.</p> +a thorough introduction of the Flink API please refer to +<a href="java_api_guide.html">Java API Programming Guide</a> or +<a href="scala_api_guide.html">Scala API Programming Guide</a>.</p> <h2 id="download">Download</h2> <p>You can download Flink from the <a href="http://flink.incubator.apache.org/downloads.html";>downloads</a> page -of the <a href="http://flink.incubator.apache.org";>project website</a>. This documentation if for version 0.7.0-incubating. Be careful +of the project website. This documentation if for version 0.6.1-incubating. Be careful when picking a version, there are different versions depending on the Hadoop and/or HDFS version that you want to use with Flink. Please refer to <a href="building.html">building</a> if you want to build Flink yourself from the source.</p> -<p>In Version 0.7.0-incubating the Scala API uses Scala 2.10. Please make +<p>In Version the Scala API uses . Please make sure to use a compatible version.</p> - - <!-- Disqus Area --> + <div style="padding-top:30px" id="disqus_thread"></div> <script type="text/javascript"> @@ -166,12 +144,14 @@ sure to use a compatible version.</p> })(); </script> <noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript";>comments powered by Disqus.</a></noscript> + <a href="http://disqus.com"; class="dsq-brlink">comments powered by <span class="logo-disqus">Disqus</span></a> + </div> </div> <div class="footer"> - <hr class="divider" /> + <p><hr class="divider"></p> <p><small>Apache Flink is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC. Incubation is @@ -182,10 +162,9 @@ incubation status is not necessarily a r stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.</small></p> -<p><a href="http://incubator.apache.org/";><img src="/img/apache-incubator-logo.png" alt="Incubator Logo" /></a></p> +<p><a href="http://incubator.apache.org/";><img src="/img/apache-incubator-logo.png" alt="Incubator Logo"></a></p> -<p class="text-center"><a href="privacy-policy.html">Privacy Policy<a> -</a></a></p> +<p class="text-center"><a href="/privacy-policy.html">Privacy Policy<a></p> </div> </div> Modified: flink/site/docs/0.6-incubating/internal_add_operator.html URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/internal_add_operator.html?rev=1658387&r1=1658386&r2=1658387&view=diff ============================================================================== --- flink/site/docs/0.6-incubating/internal_add_operator.html (original) +++ flink/site/docs/0.6-incubating/internal_add_operator.html Mon Feb 9 12:40:44 2015 @@ -5,151 +5,115 @@ <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1"> <title>Apache Flink (incubating): How to add a new Operator</title> - <link rel="shortcut icon" href="favicon.ico" type="image/x-icon"> - <link rel="icon" href="favicon.ico" type="image/x-icon"> - <link rel="stylesheet" href="css/bootstrap.css"> - <link rel="stylesheet" href="css/bootstrap-lumen-custom.css"> - <link rel="stylesheet" href="css/syntax.css"> - <link rel="stylesheet" href="css/custom.css"> - <link href="css/main/main.css" rel="stylesheet"> + <link rel="stylesheet" href="/css/bootstrap.css"> + <link rel="stylesheet" href="/css/bootstrap-lumen-custom.css"> + <link rel="stylesheet" href="/css/syntax.css"> + <link rel="/css/custom.css"> + <link rel="css/codetabs.css"> + <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.1.0/css/font-awesome.min.css" rel="stylesheet"> <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js";></script> - <script src="js/bootstrap.min.js"></script> + <script src="/js/bootstrap.min.js"></script> <script src="js/codetabs.js"></script> </head> <body> <nav class="navbar navbar-default navbar-fixed-top" role="navigation"> <div class="container"> - <div class="row"> - <div class="col-md-1 af-mobile-nav-bar"> - <a href="index.html" title="Home"> - <img class="hidden-xs hidden-sm img-responsive" - src="img/logo.png" alt="Apache Flink Logo"> - </a> - <div class="row visible-xs"> - <div class="col-xs-3"> - <a href="index.html" title="Home"> - <img class="hidden-x hidden-sm img-responsive" - src="img/logo.png" alt="Apache Flink Logo"> - </a> - </div> - <div class="col-xs-5"></div> - <div class="col-xs-4"> - <div class="af-mobile-btn"> - <span class="glyphicon glyphicon-plus"></span> - </div> - </div> - </div> - </div> - <!-- Navigation --> - <div class="col-md-11"> - <div class="collapse navbar-collapse" id="navbar-collapse-1"> - <ul class="nav navbar-nav"> - - <li> - <a href="index.html" class="">Documentation</a> - </li> - - <li> - <a href="api/java/index.html">Javadoc</a> - </li> - - <li> - <a href="api/scala/index.html#org.apache.flink.api.scala.package">Scaladoc</a> - </li> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="http://flink.incubator.apache.org";>Apache Flink</a> + </div> - </ul> - </div> - </div> + <div class="collapse navbar-collapse" id="navbar-collapse-1"> + <ul class="nav navbar-nav"> + + <li> + <a href="index.html" class="">Documentation</a> + </li> + + <li> + <a href="api/java/index.html">Javadoc</a> + </li> + </ul> </div> </div> </nav> - - <div style="padding-top:120px" class="container"> + <div style="padding-top:70px" class="container"> <div class="row"> <div class="col-md-3"> <ul> - <li><a href="faq.html">FAQ</a></li> - <li>Quickstart - <ul> - <li><a href="setup_quickstart.html">Setup</a></li> - <li><a href="run_example_quickstart.html">Run Example</a></li> - <li><a href="java_api_quickstart.html">Java API</a></li> - <li><a href="scala_api_quickstart.html">Scala API</a></li> - </ul> - </li> - - <li>Setup & Configuration - <ul> - <li><a href="local_setup.html">Local Setup</a></li> - <li><a href="building.html">Build Flink</a></li> - <li><a href="cluster_setup.html">Cluster Setup</a></li> - <li><a href="yarn_setup.html">YARN Setup</a></li> - <li><a href="config.html">Configuration</a></li> - </ul> - </li> - - <li>Programming Guides - <ul> - <li><a href="programming_guide.html">Programming Guide</a></li> - <li><a href="dataset_transformations.html">DataSet Transformations</a></li> - <li><a href="java8_programming_guide.html">Java 8 Programming Guide</a></li> - <li><a href="streaming_guide.html">Streaming Guide</a></li> - <li><a href="iterations.html">Iterations</a></li> - <li><a href="spargel_guide.html">Spargel Graph API</a></li> - <li><a href="hadoop_compatibility.html">Hadoop Compatibility</a></li> - </ul> - </li> - - <li>Examples - <ul> - <li><a href="examples.html">Bundled Examples</a></li> - <li><a href="example_connectors.html">Connecting to other systems</a></li> - </ul> - </li> - - <li>Execution - <ul> - <li><a href="local_execution.html">Local/Debugging</a></li> - <li><a href="cluster_execution.html">Cluster</a></li> - <li><a href="cli.html">Command-Line Interface</a></li> - <li><a href="web_client.html">Web Interface</a></li> - </ul> - </li> - - <li>Internals - <ul> - <li><a href="internal_overview.html">Overview</a></li> - </ul> - </li> + <li>Quickstart + <ul> + <li><a href="setup_quickstart.html">Install</a></li> + <li><a href="run_example_quickstart.html">Run Example</a></li> + <li><a href="java_api_quickstart.html">Java API</a></li> + <li><a href="scala_api_quickstart.html">Scala API</a></li> + <li><a href="faq.html">FAQ</a></li> + </ul> + </li> + + <li>Setup & Configuration + <ul> + <li><a href="building.html">Build Instructions</a></li> + <li><a href="local_setup.html">Local Setup</a></li> + <li><a href="cluster_setup.html">Cluster Setup</a></li> + <li><a href="yarn_setup.html">YARN Setup</a></li> + <li><a href="config.html">Configuration</a></li> + </ul> + </li> + + <li>Programming Guides + <ul> + <li><a href="java_api_guide.html">Java API</a></li> + <li><a href="java_api_transformations.html">Java API Transformations</a></li> + <li><a href="scala_api_guide.html">Scala API</a></li> + <li><a href="iterations.html">Iterations</a></li> + <li><a href="spargel_guide.html">Spargel Graph API</a></li> + </ul> + </li> + + <li>Examples + <ul> + <li><a href="java_api_examples.html">Java API</a></li> + <li><a href="scala_api_examples.html">Scala API</a></li> + <li><a href="example_connectors.html">Connecting to other systems</a></li> + </ul> + </li> + + <li>Execution + <ul> + <li><a href="local_execution.html">Local/Debugging</a></li> + <li><a href="cluster_execution.html">Cluster</a></li> + <li><a href="cli.html">Command-Line Interface</a></li> + <li><a href="web_client.html">Web Interface</a></li> + </ul> + </li> + + <li>Internals + <ul> + <li><a href="internal_overview.html">Overview</a></li> + <li><a href="internal_general_arch.html">General Architecture</a></li> + <li><a href="internal_add_operator.html">How-to: Adding a new Operator</a></li> + </ul> + </li> </ul> - </div> <div class="col-md-9"> <h1>How to add a new Operator</h1> - - <ul id="markdown-toc"> - <li><a href="#implementing-a-new-operator-on-dataset">Implementing a new Operator on DataSet</a> <ul> - <li><a href="#more-complex-operators">More Complex Operators</a></li> - </ul> - </li> - <li><a href="#implementing-a-custom-extension-operator">Implementing a Custom Extension Operator</a></li> - <li><a href="#implementing-a-new-runtime-operator">Implementing a new Runtime Operator</a> <ul> - <li><a href="#optimizercompiler">Optimizer/Compiler</a></li> - <li><a href="#common-api">Common API</a></li> - <li><a href="#java-api">Java API</a></li> - </ul> - </li> -</ul> -<p>Operators in the Java API can be added in multiple different ways: </p> + <p>Operators in the Java API can be added in multiple different ways: </p> <ol> - <li>On the DataSet, as a specialization/combination of existing operators</li> - <li>As a custom extension operator</li> - <li>As a new runtime operator</li> +<li>On the DataSet, as a specialization/combination of existing operators</li> +<li>As a custom extension operator</li> +<li>As a new runtime operator</li> </ol> <p>The first two approaches are typically more lightweight and easier to implement. Sometimes, @@ -159,44 +123,40 @@ new functionality does require a new run <p>Many operators can be implemented as a specialization of another operator, or by means of a UDF.</p> -<p>The simplest example are the <code>sum()</code>, <code>min()</code>, and <code>max()</code> functions on the -<a href="https://github.com/apache/incubator-flink/blob/master//flink-java/src/main/java/org/apache/flink/api/java/DataSet.java";>DataSet</a>. -These functions simply call other operations with some pre-defined parameters:</p> - -<div class="highlight"><pre><code class="language-java"><span class="kd">public</span> <span class="n">AggregateOperator</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="nf">sum</span> <span class="o">(</span><span class="kt">int</span> <span class="n">field</span><span class="o">)</span> <span class="o">{</span> - <span class="k">return</span> <span class="k">this</span><span class="o">.</span><span class="na">aggregate</span> <span class="o">(</span><span class="n">Aggregations</span><span class="o">.</span><span class="na">SUM</span><span class="o">,</span> <span class="n">field</span><span class="o">);</span> -<span class="o">}</span></code></pre></div> - +<p>The simplest example are the <code>sum()</code>, <code>min()</code>, and <code>max()</code> functions on the <a href=https://github.com/apache/incubator-flink/blob/master//flink-java/src/main/java/org/apache/flink/api/java/DataSet.java>DataSet</a>. These functions simply call other operations +with some pre-defined parameters:</p> +<div class="highlight"><pre><code class="language-text" data-lang="text">public AggregateOperator<T> sum (int field) { + return this.aggregate (Aggregations.SUM, field); +} +</code></pre></div> <p>Some operations can be implemented as compositions of multiple other operators. An example is to implement a <em>count()</em> function through a combination of <em>map</em> and <em>aggregate</em>. </p> -<p>A simple way to do this is to define a function on the <a href="https://github.com/apache/incubator-flink/blob/master//flink-java/src/main/java/org/apache/flink/api/java/DataSet.java";>DataSet</a> that calls <em>map(â¦)</em> and <em>reduce(â¦)</em> in turn:</p> - -<div class="highlight"><pre><code class="language-java"><span class="kd">public</span> <span class="n">DataSet</span><span class="o"><</span><span class="n">Long</span><span class="o">></span> <span class="nf">count</span><span class="o">()</span> <span class="o">{</span> - <span class="k">return</span> <span class="k">this</span><span class="o">.</span><span class="na">map</span><span class="o">(</span><span class="k">new</span> <span class="n">MapFunction</span><span class="o"><</span><span class="n">T</span><span class="o">,</span> <span class="n">Long</span><span class="o">>()</span> <span class="o">{</span> - <span class="kd">public</span> <span class="n">Long</span> <span class="nf">map</span><span class="o">(</span><span class="n">T</span> <span class="n">value</span><span class="o">)</span> <span class="o">{</span> - <span class="k">return</span> <span class="mi">1L</span><span class="o">;</span> - <span class="o">}</span> - <span class="o">})</span> - <span class="o">.</span><span class="na">reduce</span><span class="o">(</span><span class="k">new</span> <span class="n">ReduceFunction</span><span class="o"><</span><span class="n">Long</span><span class="o">>()</span> <span class="o">{</span> - <span class="kd">public</span> <span class="n">Long</span> <span class="nf">reduce</span><span class="o">(</span><span class="n">Long</span> <span class="n">val1</span><span class="o">,</span> <span class="n">Long</span> <span class="n">val1</span><span class="o">)</span> <span class="o">{</span> - <span class="k">return</span> <span class="n">val1</span> <span class="o">+</span> <span class="n">val2</span><span class="o">;</span> - <span class="o">}</span> - <span class="o">});</span> -<span class="o">}</span></code></pre></div> - +<p>A simple way to do this is to define a function on the <a href=https://github.com/apache/incubator-flink/blob/master//flink-java/src/main/java/org/apache/flink/api/java/DataSet.java>DataSet</a> that calls <em>map(...)</em> and <em>reduce(...)</em> in turn:</p> +<div class="highlight"><pre><code class="language-text" data-lang="text">public DataSet<Long> count() { + return this.map(new MapFunction<T, Long>() { + public Long map(T value) { + return 1L; + } + }) + .reduce(new ReduceFunction<Long>() { + public Long reduce(Long val1, Long val1) { + return val1 + val2; + } + }); +} +</code></pre></div> <p>To define a new operator without altering the DataSet class is possible by putting the functions as static members into another class. The example of the <em>count()</em> operator would look the following way:</p> - -<div class="highlight"><pre><code class="language-java"><span class="kd">public</span> <span class="kd">static</span> <span class="o"><</span><span class="n">T</span><span class="o">></span><span class="n">DataSet</span><span class="o"><</span><span class="n">Long</span><span class="o">></span> <span class="nf">count</span><span class="o">(</span><span class="n">DataSet</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="n">data</span><span class="o">)</span> <span class="o">{</span> - <span class="k">return</span> <span class="n">data</span><span class="o">.</span><span class="na">map</span><span class="o">(...).</span><span class="na">reduce</span><span class="o">(...);</span> -<span class="o">}</span></code></pre></div> - +<div class="highlight"><pre><code class="language-text" data-lang="text">public static <T>DataSet<Long> count(DataSet<T> data) { + return data.map(...).reduce(...); +} +</code></pre></div> <h3 id="more-complex-operators">More Complex Operators</h3> -<p>A more complex example of an operation via specialization is the <a href="https://github.com/apache/incubator-flink/blob/master//flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java";>Aggregation Operation</a> in the Java API. It is implemented by means of a <em>GroupReduce</em> UDF.</p> +<p>A more complex example of an operation via specialization is the <a href=https://github.com/apache/incubator-flink/blob/master//flink-java/src/main/java/org/apache/flink/api/java/operators/AggregateOperator.java>Aggregation Operation</a> in the Java API. It is implemented by means of a <em>GroupReduce</em> UDF.</p> -<p>The Aggregate Operation comes with its own operator in the <em>Java API</em>, but translates itself into a <a href="https://github.com/apache/incubator-flink/blob/master//flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java";>GroupReduceOperatorBase</a> in the <em>Common API</em>. (see <a href="internal_program_life_cycle.html">Program Life Cycle</a> for details of how an operation from the <em>Java API</em> becomes an operation of the <em>Common API</em> and finally a runtime operation.) +<p>The Aggregate Operation comes with its own operator in the <em>Java API</em>, but translates itself into a <a href=https://github.com/apache/incubator-flink/blob/master//flink-core/src/main/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorBase.java>GroupReduceOperatorBase</a> in the <em>Common API</em>. (see <a href="internal_program_life_cycle.html">Program Life Cycle</a> for details of how an operation from the <em>Java API</em> becomes an operation of the <em>Common API</em> and finally a runtime operation.) The Java API aggregation operator is only a builder that takes the types of aggregations and the field positions, and used that information to parameterize the GroupReduce UDF that performs the aggregations.</p> @@ -206,14 +166,12 @@ parameterize the GroupReduce UDF that pe <p>The DataSet offers a method for custom operators: <code>DataSet<X> runOperation(CustomUnaryOperation<T, X> operation)</code>. The <em>CustomUnaryOperation</em> interface defines operators by means of the two functions:</p> +<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kt">void</span> <span class="nf">setInput</span><span class="o">(</span><span class="n">DataSet</span><span class="o"><</span><span class="n">IN</span><span class="o">></span> <span class="n">inputData</span><span class="o">);</span> -<div class="highlight"><pre><code class="language-java"><span class="kt">void</span> <span class="nf">setInput</span><span class="o">(</span><span class="n">DataSet</span><span class="o"><</span><span class="n">IN</span><span class="o">></span> <span class="n">inputData</span><span class="o">);</span> - -<span class="n">DataSet</span><span class="o"><</span><span class="n">OUT</span><span class="o">></span> <span class="nf">createResult</span><span class="o">();</span></code></pre></div> - -<p>The <a href="https://github.com/apache/incubator-flink/blob/master//flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java";>VertexCentricIteration</a> operator is implemented that way. Below is an example how to implement the <em>count()</em> operator that way.</p> - -<div class="highlight"><pre><code class="language-java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">Counter</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="kd">implements</span> <span class="n">CustomUnaryOperation</span><span class="o"><</span><span class="n">T</span><span class="o">,</span> <span class="n">Long</span><span class="o">></span> <span class="o">{</span> +<span class="n">DataSet</span><span class="o"><</span><span class="n">OUT</span><span class="o">></span> <span class="nf">createResult</span><span class="o">();</span> +</code></pre></div> +<p>The <a href=https://github.com/apache/incubator-flink/blob/master//flink-addons/flink-spargel/src/main/java/org/apache/flink/spargel/java/VertexCentricIteration.java>VertexCentricIteration</a> operator is implemented that way. Below is an example how to implement the <em>count()</em> operator that way.</p> +<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">Counter</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="kd">implements</span> <span class="n">CustomUnaryOperation</span><span class="o"><</span><span class="n">T</span><span class="o">,</span> <span class="n">Long</span><span class="o">></span> <span class="o">{</span> <span class="kd">private</span> <span class="n">DataSet</span><span class="o"><</span><span class="n">T</span><span class="o">></span> <span class="n">input</span><span class="o">;</span> @@ -222,22 +180,21 @@ The <em>CustomUnaryOperation</em> interf <span class="kd">public</span> <span class="n">DataSet</span><span class="o"><</span><span class="n">Long</span><span class="o">></span> <span class="nf">createResult</span><span class="o">()</span> <span class="o">{</span> <span class="k">return</span> <span class="n">input</span><span class="o">.</span><span class="na">map</span><span class="o">(...).</span><span class="na">reduce</span><span class="o">(...);</span> <span class="o">}</span> -<span class="o">}</span></code></pre></div> - +<span class="o">}</span> +</code></pre></div> <p>The CountOperator can be called in the following way:</p> - -<div class="highlight"><pre><code class="language-java"><span class="n">DataSet</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">lines</span> <span class="o">=</span> <span class="o">...;</span> -<span class="n">DataSet</span><span class="o"><</span><span class="n">Long</span><span class="o">></span> <span class="n">count</span> <span class="o">=</span> <span class="n">lines</span><span class="o">.</span><span class="na">runOperation</span><span class="o">(</span><span class="k">new</span> <span class="n">Counter</span><span class="o"><</span><span class="n">String</span><span class="o">>());</span></code></pre></div> - +<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">DataSet</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">lines</span> <span class="o">=</span> <span class="o">...;</span> +<span class="n">DataSet</span><span class="o"><</span><span class="n">Long</span><span class="o">></span> <span class="n">count</span> <span class="o">=</span> <span class="n">lines</span><span class="o">.</span><span class="na">runOperation</span><span class="o">(</span><span class="k">new</span> <span class="n">Counter</span><span class="o"><</span><span class="n">String</span><span class="o">>());</span> +</code></pre></div> <h2 id="implementing-a-new-runtime-operator">Implementing a new Runtime Operator</h2> <p>Adding an new runtime operator requires changes throughout the entire stack, from the API to the runtime:</p> <ul> - <li><em>Java API</em></li> - <li><em>Common API</em></li> - <li><em>Optimizer</em></li> - <li><em>Runtime</em></li> +<li><em>Java API</em></li> +<li><em>Common API</em></li> +<li><em>Optimizer</em></li> +<li><em>Runtime</em></li> </ul> <p>We start the description bottom up, at the example of the <em>mapPartition()</em> function, which is like a <em>map</em> @@ -245,13 +202,12 @@ function, but invoked only once per para <p><strong>Runtime</strong></p> -<p>Runtime Operators are implemented using the <a href="https://github.com/apache/incubator-flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/operators/PactDriver.java";>Driver</a> interface. The interface defines the methods that describe the operator towards the runtime. The <a href="https://github.com/apache/incubator-flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/operators/MapDriver.java";>MapDriver</a> serves as a simple example of how those operators work.</p> +<p>Runtime Operators are implemented using the <a href=https://github.com/apache/incubator-flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/operators/PactDriver.java>Driver</a> interface. The interface defines the methods that describe the operator towards the runtime. The <a href=https://github.com/apache/incubator-flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/operators/MapDriver.java>MapDriver</a> serves as a simple example of how those operators work.</p> <p>The runtime works with the <code>MutableObjectIterator</code>, which describes data streams with the ability to reuse objects, to reduce pressure on the garbage collector.</p> <p>An implementation of the central <code>run()</code> method for the <em>mapPartition</em> operator could look the following way:</p> - -<div class="highlight"><pre><code class="language-java"><span class="kd">public</span> <span class="kt">void</span> <span class="nf">run</span><span class="o">()</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="o">{</span> +<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kt">void</span> <span class="nf">run</span><span class="o">()</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="o">{</span> <span class="kd">final</span> <span class="n">MutableObjectIterator</span><span class="o"><</span><span class="n">IN</span><span class="o">></span> <span class="n">input</span> <span class="o">=</span> <span class="k">this</span><span class="o">.</span><span class="na">taskContext</span><span class="o">.</span><span class="na">getInput</span><span class="o">(</span><span class="mi">0</span><span class="o">);</span> <span class="kd">final</span> <span class="n">MapPartitionFunction</span><span class="o"><</span><span class="n">IN</span><span class="o">,</span> <span class="n">OUT</span><span class="o">></span> <span class="n">function</span> <span class="o">=</span> <span class="k">this</span><span class="o">.</span><span class="na">taskContext</span><span class="o">.</span><span class="na">getStub</span><span class="o">();</span> <span class="kd">final</span> <span class="n">Collector</span><span class="o"><</span><span class="n">OUT</span><span class="o">></span> <span class="n">output</span> <span class="o">=</span> <span class="k">this</span><span class="o">.</span><span class="na">taskContext</span><span class="o">.</span><span class="na">getOutputCollector</span><span class="o">();</span> @@ -261,97 +217,85 @@ function, but invoked only once per para <span class="n">Iterator</span><span class="o"><</span><span class="n">IN</span><span class="o">></span> <span class="n">iterator</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">MutableToRegularIteratorWrapper</span><span class="o">(</span><span class="n">input</span><span class="o">,</span> <span class="n">serializer</span><span class="o">);</span> <span class="n">function</span><span class="o">.</span><span class="na">mapPartition</span><span class="o">(</span><span class="n">iterator</span><span class="o">,</span> <span class="n">output</span><span class="o">);</span> -<span class="o">}</span></code></pre></div> - +<span class="o">}</span> +</code></pre></div> <p>To increase efficiency, it is often beneficial to implement a <em>chained</em> version of an operator. Chained operators run in the same thread as their preceding operator, and work with nested function calls. This is very efficient, because it saves serialization/deserialization overhead.</p> -<p>To learn how to implement a chained operator, take a look at the <a href="https://github.com/apache/incubator-flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/operators/MapDriver.java";>MapDriver</a> (regular) and the -<a href="https://github.com/apache/incubator-flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedMapDriver.java";>ChainedMapDriver</a> (chained variant).</p> +<p>To learn how to implement a chained operator, take a look at the <a href=https://github.com/apache/incubator-flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/operators/MapDriver.java>MapDriver</a> (regular) and the +<a href=https://github.com/apache/incubator-flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedMapDriver.java>ChainedMapDriver</a> (chained variant).</p> -<h3 id="optimizercompiler">Optimizer/Compiler</h3> +<p><strong>Optimizer/Compiler</strong></p> <p>This section does a minimal discussion of the important steps to add an operator. Please see the <a href="internal_optimizer.html">Optimizer</a> docs for more detail on how the optimizer works. To allow the optimizer to include a new operator in its planning, it needs a bit of information about it; in particular, the following information:</p> <ul> - <li> - <p><em><a href="https://github.com/apache/incubator-flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/operators/DriverStrategy.java";>DriverStrategy</a></em>: The operation needs to be added to the Enum, to make it available to the optimizer. The parameters to the Enum entry define which class implements the runtime operator, its chained version, whether the operator accumulates records (and needs memory for that), and whether it requires a comparator (works on keys). For our example, we can add the entry -~~~ java -MAP_PARTITION(MapPartitionDriver.class, null /* or chained variant */, PIPELINED, false) -~~~</p> - </li> - <li> - <p><em>Cost function</em>: The class <a href="https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/costs/CostEstimator.java";>CostEstimator</a> needs to know how expensive the operation is to the system. The costs here refer to the non-UDF part of the operator. Since the operator does essentially no work (it forwards the record stream to the UDF), the costs are zero. We change the <code>costOperator(...)</code> method by adding the <em>MAP_PARTITION</em> constant to the switch statement similar to the <em>MAP</em> constant such that no cost is accounted for it.</p> - </li> - <li> - <p><em>OperatorDescriptor</em>: The operator descriptors define how an operation needs to be treated by the optimizer. They describe how the operation requires the input data to be (e.g., sorted or partitioned) and that way allows the optimizer to optimize the data movement, sorting, grouping in a global fashion. They do that by describing which <a href="https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/dataproperties/RequestedGlobalProperties.java";>RequestedGlobalProperties</a> (partitioning, replication, etc) and which <a href="https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/dataproperties/RequestedLocalProperties.java";>RequestedLocalProperties</a> (sorting, grouping, uniqueness) the operator has, as well as how the operator affects the existing <a href="https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/com piler/dataproperties/GlobalProperties.java">GlobalProperties</a> and <a href="https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/dataproperties/LocalProperties.java";>LocalProperties</a>. In addition, it defines a few utility methods, for example to instantiate an operator candidate. -Since the <em>mapPartition()</em> function is very simple (no requirements on partitioning/grouping), the descriptor is very simple. Other operators have more complex requirements, for example the <a href="https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/operators/GroupReduceProperties.java";>GroupReduce</a>. Some operators, like <em>join</em> have multiple ways in which they can be executed and therefore have multiple descriptors (<a href="https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/operators/HashJoinBuildFirstProperties.java";>Hash Join 1</a>, <a href="https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/operators/HashJoinBuildSecondProperties.java";>Hash Join 2</a>, <a href="https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/operators/SortMerge JoinDescriptor.java">SortMerge Join</a>). -The code sample below explains (with comments) how to create a descriptor for the <em>MapPartitionOperator</em> </p> - - <div class="highlight"><pre><code class="language-java"> <span class="kd">public</span> <span class="n">DriverStrategy</span> <span class="nf">getStrategy</span><span class="o">()</span> <span class="o">{</span> - <span class="k">return</span> <span class="n">MAP_PARTITION</span><span class="o">;</span> - <span class="o">}</span> - - <span class="c1">// Instantiate the operator with the strategy over the input given in the form of the Channel</span> - <span class="kd">public</span> <span class="n">SingleInputPlanNode</span> <span class="nf">instantiate</span><span class="o">(</span><span class="n">Channel</span> <span class="n">in</span><span class="o">,</span> <span class="n">SingleInputNode</span> <span class="n">node</span><span class="o">)</span> <span class="o">{</span> - <span class="k">return</span> <span class="k">new</span> <span class="nf">SingleInputPlanNode</span><span class="o">(</span><span class="n">node</span><span class="o">,</span> <span class="s">"MapPartition"</span><span class="o">,</span> <span class="n">in</span><span class="o">,</span> <span class="n">MAP_PARTITION</span><span class="o">);</span> - <span class="o">}</span> - - <span class="c1">// The operation accepts data with default global properties (arbitrary distribution)</span> - <span class="kd">protected</span> <span class="n">List</span><span class="o"><</span><span class="n">RequestedGlobalProperties</span><span class="o">></span> <span class="nf">createPossibleGlobalProperties</span><span class="o">()</span> <span class="o">{</span> - <span class="k">return</span> <span class="n">Collections</span><span class="o">.</span><span class="na">singletonList</span><span class="o">(</span><span class="k">new</span> <span class="nf">RequestedGlobalProperties</span><span class="o">());</span> - <span class="o">}</span> - - <span class="c1">// The operation can accept data with any local properties. No grouping/sorting is necessary</span> - <span class="kd">protected</span> <span class="n">List</span><span class="o"><</span><span class="n">RequestedLocalProperties</span><span class="o">></span> <span class="nf">createPossibleLocalProperties</span><span class="o">()</span> <span class="o">{</span> - <span class="k">return</span> <span class="n">Collections</span><span class="o">.</span><span class="na">singletonList</span><span class="o">(</span><span class="k">new</span> <span class="nf">RequestedLocalProperties</span><span class="o">());</span> - <span class="o">}</span> - - <span class="c1">// the operation itself does not affect the existing global properties.</span> - <span class="c1">// The effect of the UDF's semantics// are evaluated separately (by interpreting the</span> - <span class="c1">// semantic assertions)</span> - <span class="kd">public</span> <span class="n">GlobalProperties</span> <span class="nf">computeGlobalProperties</span><span class="o">(</span><span class="n">GlobalProperties</span> <span class="n">gProps</span><span class="o">)</span> <span class="o">{</span> - <span class="k">return</span> <span class="n">gProps</span><span class="o">;</span> - <span class="o">}</span> - - <span class="c1">// since the operation can mess up all order, grouping, uniqueness, we cannot make any statements</span> - <span class="c1">// about how local properties are preserved</span> - <span class="kd">public</span> <span class="n">LocalProperties</span> <span class="nf">computeLocalProperties</span><span class="o">(</span><span class="n">LocalProperties</span> <span class="n">lProps</span><span class="o">)</span> <span class="o">{</span> - <span class="k">return</span> <span class="n">LocalProperties</span><span class="o">.</span><span class="na">EMPTY</span><span class="o">;</span> - <span class="o">}</span></code></pre></div> - </li> - <li> - <p><em>OptimizerNode</em>: The optimizer node is the place where all comes together. It creates the list of <em>OperatorDescriptors</em>, implements the result data set size estimation, and assigns a name to the operation. It is a relatively small class and can be more or less copied again from the <a href="https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/dag/MapNode.java";>MapNode</a>.</p> - </li> +<li><p><em><a href=https://github.com/apache/incubator-flink/blob/master//flink-runtime/src/main/java/org/apache/flink/runtime/operators/DriverStrategy.java>DriverStrategy</a></em>: The operation needs to be added to the Enum, to make it available to the optimizer. The parameters to the Enum entry define which class implements the runtime operator, its chained version, whether the operator accumulates records (and needs memory for that), and whether it requires a comparator (works on keys). For our example, we can add the entry</p> +<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">MAP_PARTITION</span><span class="o">(</span><span class="n">MapPartitionDriver</span><span class="o">.</span><span class="na">class</span><span class="o">,</span> <span class="kc">null</span> <span class="cm">/* or chained variant */</span><span class="o">,</span> <span class="n">PIPELINED</span><span class="o">,</span> <span class="kc">false</span><span class="o">)</span> +</code></pre></div></li> +<li><p><em>Cost function</em>: The class <a href=https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/costs/CostEstimator.java>CostEstimator</a> needs to know how expensive the operation is to the system. The costs here refer to the non-UDF part of the operator. Since the operator does essentially no work (it forwards the record stream to the UDF), the costs are zero. We change the <code>costOperator(...)</code> method by adding the <em>MAP_PARTITION</em> constant to the switch statement similar to the <em>MAP</em> constant such that no cost is accounted for it.</p></li> +<li><p><em>OperatorDescriptor</em>: The operator descriptors define how an operation needs to be treated by the optimizer. They describe how the operation requires the input data to be (e.g., sorted or partitioned) and that way allows the optimizer to optimize the data movement, sorting, grouping in a global fashion. They do that by describing which <a href=https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/dataproperties/RequestedGlobalProperties.java>RequestedGlobalProperties</a> (partitioning, replication, etc) and which <a href=https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/dataproperties/RequestedLocalProperties.java>RequestedLocalProperties</a> (sorting, grouping, uniqueness) the operator has, as well as how the operator affects the existing <a href=https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler /dataproperties/GlobalProperties.java>GlobalProperties</a> and <a href=https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/dataproperties/LocalProperties.java>LocalProperties</a>. In addition, it defines a few utility methods, for example to instantiate an operator candidate. +Since the <em>mapPartition()</em> function is very simple (no requirements on partitioning/grouping), the descriptor is very simple. Other operators have more complex requirements, for example the <a href=https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/operators/GroupReduceProperties.java>GroupReduce</a>. Some operators, like <em>join</em> have multiple ways in which they can be executed and therefore have multiple descriptors (<a href=https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/operators/HashJoinBuildFirstProperties.java>Hash Join 1</a>, <a href=https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/operators/HashJoinBuildSecondProperties.java>Hash Join 2</a>, <a href=https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/operators/SortMergeJoinDes criptor.java>SortMerge Join</a>). +The code sample below explains (with comments) how to create a descriptor for the <em>MapPartitionOperator</em></p> +<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="n">DriverStrategy</span> <span class="nf">getStrategy</span><span class="o">()</span> <span class="o">{</span> + <span class="k">return</span> <span class="n">MAP_PARTITION</span><span class="o">;</span> +<span class="o">}</span> + +<span class="c1">// Instantiate the operator with the strategy over the input given in the form of the Channel</span> +<span class="kd">public</span> <span class="n">SingleInputPlanNode</span> <span class="nf">instantiate</span><span class="o">(</span><span class="n">Channel</span> <span class="n">in</span><span class="o">,</span> <span class="n">SingleInputNode</span> <span class="n">node</span><span class="o">)</span> <span class="o">{</span> + <span class="k">return</span> <span class="k">new</span> <span class="nf">SingleInputPlanNode</span><span class="o">(</span><span class="n">node</span><span class="o">,</span> <span class="s">"MapPartition"</span><span class="o">,</span> <span class="n">in</span><span class="o">,</span> <span class="n">MAP_PARTITION</span><span class="o">);</span> +<span class="o">}</span> + +<span class="c1">// The operation accepts data with default global properties (arbitrary distribution)</span> +<span class="kd">protected</span> <span class="n">List</span><span class="o"><</span><span class="n">RequestedGlobalProperties</span><span class="o">></span> <span class="nf">createPossibleGlobalProperties</span><span class="o">()</span> <span class="o">{</span> + <span class="k">return</span> <span class="n">Collections</span><span class="o">.</span><span class="na">singletonList</span><span class="o">(</span><span class="k">new</span> <span class="nf">RequestedGlobalProperties</span><span class="o">());</span> +<span class="o">}</span> + +<span class="c1">// The operation can accept data with any local properties. No grouping/sorting is necessary</span> +<span class="kd">protected</span> <span class="n">List</span><span class="o"><</span><span class="n">RequestedLocalProperties</span><span class="o">></span> <span class="nf">createPossibleLocalProperties</span><span class="o">()</span> <span class="o">{</span> + <span class="k">return</span> <span class="n">Collections</span><span class="o">.</span><span class="na">singletonList</span><span class="o">(</span><span class="k">new</span> <span class="nf">RequestedLocalProperties</span><span class="o">());</span> +<span class="o">}</span> + +<span class="c1">// the operation itself does not affect the existing global properties.</span> +<span class="c1">// The effect of the UDF's semantics// are evaluated separately (by interpreting the</span> +<span class="c1">// semantic assertions)</span> +<span class="kd">public</span> <span class="n">GlobalProperties</span> <span class="nf">computeGlobalProperties</span><span class="o">(</span><span class="n">GlobalProperties</span> <span class="n">gProps</span><span class="o">)</span> <span class="o">{</span> + <span class="k">return</span> <span class="n">gProps</span><span class="o">;</span> +<span class="o">}</span> + +<span class="c1">// since the operation can mess up all order, grouping, uniqueness, we cannot make any statements</span> +<span class="c1">// about how local properties are preserved</span> +<span class="kd">public</span> <span class="n">LocalProperties</span> <span class="nf">computeLocalProperties</span><span class="o">(</span><span class="n">LocalProperties</span> <span class="n">lProps</span><span class="o">)</span> <span class="o">{</span> + <span class="k">return</span> <span class="n">LocalProperties</span><span class="o">.</span><span class="na">EMPTY</span><span class="o">;</span> +<span class="o">}</span> +</code></pre></div></li> +<li><p><em>OptimizerNode</em>: The optimizer node is the place where all comes together. It creates the list of <em>OperatorDescriptors</em>, implements the result data set size estimation, and assigns a name to the operation. It is a relatively small class and can be more or less copied again from the <a href=https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/dag/MapNode.java>MapNode</a>.</p></li> </ul> -<h3 id="common-api">Common API</h3> +<p><strong>Common API</strong></p> <p>To make the operation available to the higher-level APIs, it needs to be added to the Common API. The simplest way to do this is to add a -base operator. Create a class <code>MapPartitionOperatorBase</code>, after the pattern of the <a href="https://github.com/apache/incubator-flink/blob/master//flink-core/src/main/java/org/apache/flink/api/common/operators/base/MapOperatorBase.java";>MapOperatorBase</a>.</p> +base operator. Create a class <code>MapPartitionOperatorBase</code>, after the pattern of the <a href=https://github.com/apache/incubator-flink/blob/master//flink-core/src/main/java/org/apache/flink/api/common/operators/base/MapOperatorBase.java>MapOperatorBase</a>.</p> <p>In addition, the optimizer needs to know which OptimizerNode how to create an OptimizerNode from the OperatorBase. This happens in the class -<code>GraphCreatingVisitor</code> in the <a href="https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/PactCompiler.java";>Optimizer</a>.</p> +<code>GraphCreatingVisitor</code> in the <a href=https://github.com/apache/incubator-flink/blob/master//flink-compiler/src/main/java/org/apache/flink/compiler/PactCompiler.java>Optimizer</a>.</p> <p><em>Note:</em> A pending idea is to allow to skip this step by unifying the OptimizerNode and the Common API operator. They essentially fulfill the same function. The Common API operator exists only in order for the <code>flink-java</code> and <code>flink-scala</code> packages to not have a dependency on the optimizer.</p> -<h3 id="java-api">Java API</h3> +<p><strong>Java API</strong></p> -<p>Create a Java API operator that is constructed in the same way as the <a href="https://github.com/apache/incubator-flink/blob/master//flink-java/src/main/java/org/apache/flink/api/java/operators/MapOperator.java";>MapOperator</a>. The core method is the <code>translateToDataFlow(...)</code> method, which creates the Common API operator for the Java API operator.</p> +<p>Create a Java API operator that is constructed in the same way as the <a href=https://github.com/apache/incubator-flink/blob/master//flink-java/src/main/java/org/apache/flink/api/java/operators/MapOperator.java>MapOperator</a>. The core method is the <code>translateToDataFlow(...)</code> method, which creates the Common API operator for the Java API operator.</p> <p>The final step is to add a function to the <code>DataSet</code> class:</p> - -<div class="highlight"><pre><code class="language-java"><span class="kd">public</span> <span class="o"><</span><span class="n">R</span><span class="o">></span> <span class="n">DataSet</span><span class="o"><</span><span class="n">R</span><span class="o">></span> <span class="nf">mapPartition</span><span class="o">(</span><span class="n">MapPartitionFunction</span><span class="o"><</span><span class="n">T</span><span class="o">,</span> <span class="n">R</span><span class="o">></span> <span class="n">function</span><span class="o">)</span> <span class="o">{</span> +<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="o"><</span><span class="n">R</span><span class="o">></span> <span class="n">DataSet</span><span class="o"><</span><span class="n">R</span><span class="o">></span> <span class="nf">mapPartition</span><span class="o">(</span><span class="n">MapPartitionFunction</span><span class="o"><</span><span class="n">T</span><span class="o">,</span> <span class="n">R</span><span class="o">></span> <span class="n">function</span><span class="o">)</span> <span class="o">{</span> <span class="k">return</span> <span class="k">new</span> <span class="n">MapPartitionOperator</span><span class="o"><</span><span class="n">T</span><span class="o">,</span> <span class="n">R</span><span class="o">>(</span><span class="k">this</span><span class="o">,</span> <span class="n">function</span><span class="o">);</span> -<span class="o">}</span></code></pre></div> +<span class="o">}</span> +</code></pre></div> - - - <!-- Disqus Area --> <div style="padding-top:30px" id="disqus_thread"></div> <script type="text/javascript"> @@ -366,12 +310,14 @@ optimizer.</p> })(); </script> <noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript";>comments powered by Disqus.</a></noscript> + <a href="http://disqus.com"; class="dsq-brlink">comments powered by <span class="logo-disqus">Disqus</span></a> + </div> </div> <div class="footer"> - <hr class="divider" /> + <p><hr class="divider"></p> <p><small>Apache Flink is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC. Incubation is @@ -382,10 +328,9 @@ incubation status is not necessarily a r stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.</small></p> -<p><a href="http://incubator.apache.org/";><img src="/img/apache-incubator-logo.png" alt="Incubator Logo" /></a></p> +<p><a href="http://incubator.apache.org/";><img src="/img/apache-incubator-logo.png" alt="Incubator Logo"></a></p> -<p class="text-center"><a href="privacy-policy.html">Privacy Policy<a> -</a></a></p> +<p class="text-center"><a href="/privacy-policy.html">Privacy Policy<a></p> </div> </div> Modified: flink/site/docs/0.6-incubating/internal_distributed_runtime.html URL: http://svn.apache.org/viewvc/flink/site/docs/0.6-incubating/internal_distributed_runtime.html?rev=1658387&r1=1658386&r2=1658387&view=diff ============================================================================== --- flink/site/docs/0.6-incubating/internal_distributed_runtime.html (original) +++ flink/site/docs/0.6-incubating/internal_distributed_runtime.html Mon Feb 9 12:40:44 2015 @@ -5,135 +5,112 @@ <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1"> <title>Apache Flink (incubating): Distributed Runtime</title> - <link rel="shortcut icon" href="favicon.ico" type="image/x-icon"> - <link rel="icon" href="favicon.ico" type="image/x-icon"> - <link rel="stylesheet" href="css/bootstrap.css"> - <link rel="stylesheet" href="css/bootstrap-lumen-custom.css"> - <link rel="stylesheet" href="css/syntax.css"> - <link rel="stylesheet" href="css/custom.css"> - <link href="css/main/main.css" rel="stylesheet"> + <link rel="stylesheet" href="/css/bootstrap.css"> + <link rel="stylesheet" href="/css/bootstrap-lumen-custom.css"> + <link rel="stylesheet" href="/css/syntax.css"> + <link rel="/css/custom.css"> + <link rel="css/codetabs.css"> + <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.1.0/css/font-awesome.min.css" rel="stylesheet"> <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js";></script> - <script src="js/bootstrap.min.js"></script> + <script src="/js/bootstrap.min.js"></script> <script src="js/codetabs.js"></script> </head> <body> <nav class="navbar navbar-default navbar-fixed-top" role="navigation"> <div class="container"> - <div class="row"> - <div class="col-md-1 af-mobile-nav-bar"> - <a href="index.html" title="Home"> - <img class="hidden-xs hidden-sm img-responsive" - src="img/logo.png" alt="Apache Flink Logo"> - </a> - <div class="row visible-xs"> - <div class="col-xs-3"> - <a href="index.html" title="Home"> - <img class="hidden-x hidden-sm img-responsive" - src="img/logo.png" alt="Apache Flink Logo"> - </a> - </div> - <div class="col-xs-5"></div> - <div class="col-xs-4"> - <div class="af-mobile-btn"> - <span class="glyphicon glyphicon-plus"></span> - </div> - </div> - </div> - </div> - <!-- Navigation --> - <div class="col-md-11"> - <div class="collapse navbar-collapse" id="navbar-collapse-1"> - <ul class="nav navbar-nav"> - - <li> - <a href="index.html" class="">Documentation</a> - </li> - - <li> - <a href="api/java/index.html">Javadoc</a> - </li> - - <li> - <a href="api/scala/index.html#org.apache.flink.api.scala.package">Scaladoc</a> - </li> + <div class="navbar-header"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="http://flink.incubator.apache.org";>Apache Flink</a> + </div> - </ul> - </div> - </div> + <div class="collapse navbar-collapse" id="navbar-collapse-1"> + <ul class="nav navbar-nav"> + + <li> + <a href="index.html" class="">Documentation</a> + </li> + + <li> + <a href="api/java/index.html">Javadoc</a> + </li> + </ul> </div> </div> </nav> - - <div style="padding-top:120px" class="container"> + <div style="padding-top:70px" class="container"> <div class="row"> <div class="col-md-3"> <ul> - <li><a href="faq.html">FAQ</a></li> - <li>Quickstart - <ul> - <li><a href="setup_quickstart.html">Setup</a></li> - <li><a href="run_example_quickstart.html">Run Example</a></li> - <li><a href="java_api_quickstart.html">Java API</a></li> - <li><a href="scala_api_quickstart.html">Scala API</a></li> - </ul> - </li> - - <li>Setup & Configuration - <ul> - <li><a href="local_setup.html">Local Setup</a></li> - <li><a href="building.html">Build Flink</a></li> - <li><a href="cluster_setup.html">Cluster Setup</a></li> - <li><a href="yarn_setup.html">YARN Setup</a></li> - <li><a href="config.html">Configuration</a></li> - </ul> - </li> - - <li>Programming Guides - <ul> - <li><a href="programming_guide.html">Programming Guide</a></li> - <li><a href="dataset_transformations.html">DataSet Transformations</a></li> - <li><a href="java8_programming_guide.html">Java 8 Programming Guide</a></li> - <li><a href="streaming_guide.html">Streaming Guide</a></li> - <li><a href="iterations.html">Iterations</a></li> - <li><a href="spargel_guide.html">Spargel Graph API</a></li> - <li><a href="hadoop_compatibility.html">Hadoop Compatibility</a></li> - </ul> - </li> - - <li>Examples - <ul> - <li><a href="examples.html">Bundled Examples</a></li> - <li><a href="example_connectors.html">Connecting to other systems</a></li> - </ul> - </li> - - <li>Execution - <ul> - <li><a href="local_execution.html">Local/Debugging</a></li> - <li><a href="cluster_execution.html">Cluster</a></li> - <li><a href="cli.html">Command-Line Interface</a></li> - <li><a href="web_client.html">Web Interface</a></li> - </ul> - </li> - - <li>Internals - <ul> - <li><a href="internal_overview.html">Overview</a></li> - </ul> - </li> + <li>Quickstart + <ul> + <li><a href="setup_quickstart.html">Install</a></li> + <li><a href="run_example_quickstart.html">Run Example</a></li> + <li><a href="java_api_quickstart.html">Java API</a></li> + <li><a href="scala_api_quickstart.html">Scala API</a></li> + <li><a href="faq.html">FAQ</a></li> + </ul> + </li> + + <li>Setup & Configuration + <ul> + <li><a href="building.html">Build Instructions</a></li> + <li><a href="local_setup.html">Local Setup</a></li> + <li><a href="cluster_setup.html">Cluster Setup</a></li> + <li><a href="yarn_setup.html">YARN Setup</a></li> + <li><a href="config.html">Configuration</a></li> + </ul> + </li> + + <li>Programming Guides + <ul> + <li><a href="java_api_guide.html">Java API</a></li> + <li><a href="java_api_transformations.html">Java API Transformations</a></li> + <li><a href="scala_api_guide.html">Scala API</a></li> + <li><a href="iterations.html">Iterations</a></li> + <li><a href="spargel_guide.html">Spargel Graph API</a></li> + </ul> + </li> + + <li>Examples + <ul> + <li><a href="java_api_examples.html">Java API</a></li> + <li><a href="scala_api_examples.html">Scala API</a></li> + <li><a href="example_connectors.html">Connecting to other systems</a></li> + </ul> + </li> + + <li>Execution + <ul> + <li><a href="local_execution.html">Local/Debugging</a></li> + <li><a href="cluster_execution.html">Cluster</a></li> + <li><a href="cli.html">Command-Line Interface</a></li> + <li><a href="web_client.html">Web Interface</a></li> + </ul> + </li> + + <li>Internals + <ul> + <li><a href="internal_overview.html">Overview</a></li> + <li><a href="internal_general_arch.html">General Architecture</a></li> + <li><a href="internal_add_operator.html">How-to: Adding a new Operator</a></li> + </ul> + </li> </ul> - </div> <div class="col-md-9"> <h1>Distributed Runtime</h1> - - <p>Pendingâ¦</p> - - <!-- Disqus Area --> + <p>Pending...</p> + + <div style="padding-top:30px" id="disqus_thread"></div> <script type="text/javascript"> @@ -148,12 +125,14 @@ })(); </script> <noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript";>comments powered by Disqus.</a></noscript> + <a href="http://disqus.com"; class="dsq-brlink">comments powered by <span class="logo-disqus">Disqus</span></a> + </div> </div> <div class="footer"> - <hr class="divider" /> + <p><hr class="divider"></p> <p><small>Apache Flink is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC. Incubation is @@ -164,10 +143,9 @@ incubation status is not necessarily a r stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.</small></p> -<p><a href="http://incubator.apache.org/";><img src="/img/apache-incubator-logo.png" alt="Incubator Logo" /></a></p> +<p><a href="http://incubator.apache.org/";><img src="/img/apache-incubator-logo.png" alt="Incubator Logo"></a></p> -<p class="text-center"><a href="privacy-policy.html">Privacy Policy<a> -</a></a></p> +<p class="text-center"><a href="/privacy-policy.html">Privacy Policy<a></p> </div> </div>
