[FLINK-4317, FLIP-3] [docs] Restructure docs - Add redirect layout - Remove Maven artifact name warning - Add info box if stable, but not latest - Add font-awesome 4.6.3 - Add sidenav layout
This closes #2387. Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/844c874b Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/844c874b Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/844c874b Branch: refs/heads/flip-6 Commit: 844c874b52546eaca78af8cbfc6f08cb2b4d873c Parents: c09ff03 Author: Ufuk Celebi <u...@apache.org> Authored: Wed Aug 17 15:06:04 2016 +0200 Committer: Ufuk Celebi <u...@apache.org> Committed: Wed Aug 24 11:25:56 2016 +0200 ---------------------------------------------------------------------- LICENSE | 5 +- docs/README.md | 42 +- docs/_config.yml | 17 +- docs/_includes/navbar.html | 117 - docs/_includes/sidenav.html | 149 ++ docs/_layouts/base.html | 33 +- docs/_layouts/plain.html | 130 +- docs/_layouts/redirect.html | 27 + docs/apis/batch/connectors.md | 242 -- docs/apis/batch/dataset_transformations.md | 2338 ------------------ docs/apis/batch/examples.md | 521 ---- docs/apis/batch/fault_tolerance.md | 100 - docs/apis/batch/fig/LICENSE.txt | 17 - .../fig/iterations_delta_iterate_operator.png | Bin 113607 -> 0 bytes ...terations_delta_iterate_operator_example.png | Bin 335057 -> 0 bytes .../batch/fig/iterations_iterate_operator.png | Bin 63465 -> 0 bytes .../fig/iterations_iterate_operator_example.png | Bin 102925 -> 0 bytes docs/apis/batch/fig/iterations_supersteps.png | Bin 54098 -> 0 bytes docs/apis/batch/hadoop_compatibility.md | 249 -- docs/apis/batch/index.md | 2274 ----------------- docs/apis/batch/iterations.md | 213 -- docs/apis/batch/libs/fig/LICENSE.txt | 17 - .../apis/batch/libs/fig/gelly-example-graph.png | Bin 18813 -> 0 bytes docs/apis/batch/libs/fig/gelly-filter.png | Bin 57192 -> 0 bytes docs/apis/batch/libs/fig/gelly-gsa-sssp1.png | Bin 31201 -> 0 bytes .../apis/batch/libs/fig/gelly-reduceOnEdges.png | Bin 23843 -> 0 bytes .../batch/libs/fig/gelly-reduceOnNeighbors.png | Bin 34903 -> 0 bytes docs/apis/batch/libs/fig/gelly-union.png | Bin 50498 -> 0 bytes docs/apis/batch/libs/fig/gelly-vc-sssp1.png | Bin 28537 -> 0 bytes .../libs/fig/vertex-centric supersteps.png | Bin 80198 -> 0 bytes docs/apis/batch/libs/gelly.md | 26 - docs/apis/batch/libs/gelly/graph_algorithms.md | 311 --- docs/apis/batch/libs/gelly/graph_api.md | 836 ------- docs/apis/batch/libs/gelly/graph_generators.md | 657 ----- docs/apis/batch/libs/gelly/index.md | 74 - .../libs/gelly/iterative_graph_processing.md | 971 -------- docs/apis/batch/libs/gelly/library_methods.md | 350 --- docs/apis/batch/libs/index.md | 29 - docs/apis/batch/libs/ml/als.md | 178 -- docs/apis/batch/libs/ml/contribution_guide.md | 110 - docs/apis/batch/libs/ml/cross_validation.md | 175 -- docs/apis/batch/libs/ml/distance_metrics.md | 111 - docs/apis/batch/libs/ml/index.md | 151 -- docs/apis/batch/libs/ml/knn.md | 149 -- docs/apis/batch/libs/ml/min_max_scaler.md | 116 - .../batch/libs/ml/multiple_linear_regression.md | 164 -- docs/apis/batch/libs/ml/optimization.md | 385 --- docs/apis/batch/libs/ml/pipelines.md | 445 ---- docs/apis/batch/libs/ml/polynomial_features.md | 111 - docs/apis/batch/libs/ml/quickstart.md | 244 -- docs/apis/batch/libs/ml/standard_scaler.md | 116 - docs/apis/batch/libs/ml/svm.md | 223 -- docs/apis/batch/libs/table.md | 26 - docs/apis/batch/python.md | 638 ----- docs/apis/batch/zip_elements_guide.md | 128 - docs/apis/best_practices.md | 403 --- docs/apis/cli.md | 322 --- docs/apis/cluster_execution.md | 156 -- docs/apis/common/fig/plan_visualizer.png | Bin 145778 -> 0 bytes docs/apis/common/index.md | 1352 ---------- docs/apis/connectors.md | 23 - docs/apis/index.md | 21 - docs/apis/java8.md | 198 -- docs/apis/local_execution.md | 126 - docs/apis/metrics.md | 470 ---- docs/apis/programming_guide.md | 26 - docs/apis/scala_api_extensions.md | 409 --- docs/apis/scala_shell.md | 197 -- docs/apis/streaming/connectors/cassandra.md | 158 -- docs/apis/streaming/connectors/elasticsearch.md | 183 -- .../apis/streaming/connectors/elasticsearch2.md | 144 -- .../streaming/connectors/filesystem_sink.md | 133 - docs/apis/streaming/connectors/index.md | 47 - docs/apis/streaming/connectors/kafka.md | 293 --- docs/apis/streaming/connectors/kinesis.md | 322 --- docs/apis/streaming/connectors/nifi.md | 141 -- docs/apis/streaming/connectors/rabbitmq.md | 132 - docs/apis/streaming/connectors/redis.md | 177 -- docs/apis/streaming/connectors/twitter.md | 89 - docs/apis/streaming/event_time.md | 208 -- .../streaming/event_timestamp_extractors.md | 108 - .../streaming/event_timestamps_watermarks.md | 332 --- docs/apis/streaming/fault_tolerance.md | 462 ---- docs/apis/streaming/fig/LICENSE.txt | 17 - .../fig/parallel_streams_watermarks.svg | 516 ---- docs/apis/streaming/fig/rescale.svg | 472 ---- docs/apis/streaming/fig/savepoints-overview.png | Bin 62824 -> 0 bytes .../streaming/fig/savepoints-program_ids.png | Bin 55492 -> 0 bytes .../streaming/fig/stream_watermark_in_order.svg | 314 --- .../fig/stream_watermark_out_of_order.svg | 314 --- docs/apis/streaming/fig/times_clocks.svg | 368 --- docs/apis/streaming/index.md | 1787 ------------- docs/apis/streaming/libs/cep.md | 659 ----- docs/apis/streaming/libs/index.md | 27 - docs/apis/streaming/non-windowed.svg | 22 - docs/apis/streaming/savepoints.md | 110 - docs/apis/streaming/session-windows.svg | 22 - docs/apis/streaming/sliding-windows.svg | 22 - docs/apis/streaming/state.md | 295 --- docs/apis/streaming/state_backends.md | 163 -- docs/apis/streaming/storm_compatibility.md | 287 --- docs/apis/streaming/tumbling-windows.svg | 22 - docs/apis/streaming/windows.md | 678 ----- docs/apis/streaming_guide.md | 26 - docs/apis/table.md | 2082 ---------------- docs/concepts/concepts.md | 246 -- docs/concepts/fig/checkpoints.svg | 249 -- .../fig/event_ingestion_processing_time.svg | 375 --- docs/concepts/fig/parallel_dataflow.svg | 487 ---- docs/concepts/fig/processes.svg | 749 ------ docs/concepts/fig/program_dataflow.svg | 546 ---- docs/concepts/fig/slot_sharing.svg | 721 ------ docs/concepts/fig/state_partitioning.svg | 291 --- docs/concepts/fig/tasks_chains.svg | 463 ---- docs/concepts/fig/tasks_slots.svg | 395 --- docs/concepts/fig/windows.svg | 193 -- docs/concepts/index.md | 249 ++ docs/dev/api_concepts.md | 1349 ++++++++++ docs/dev/apis.md | 24 + docs/dev/batch/connectors.md | 238 ++ docs/dev/batch/dataset_transformations.md | 2335 +++++++++++++++++ docs/dev/batch/examples.md | 519 ++++ docs/dev/batch/fault_tolerance.md | 98 + docs/dev/batch/hadoop_compatibility.md | 248 ++ docs/dev/batch/index.md | 2267 +++++++++++++++++ docs/dev/batch/iterations.md | 212 ++ docs/dev/batch/python.md | 635 +++++ docs/dev/batch/zip_elements_guide.md | 126 + docs/dev/cluster_execution.md | 155 ++ docs/dev/connectors/cassandra.md | 155 ++ docs/dev/connectors/elasticsearch.md | 180 ++ docs/dev/connectors/elasticsearch2.md | 141 ++ docs/dev/connectors/filesystem_sink.md | 130 + docs/dev/connectors/index.md | 46 + docs/dev/connectors/kafka.md | 289 +++ docs/dev/connectors/kinesis.md | 319 +++ docs/dev/connectors/nifi.md | 138 ++ docs/dev/connectors/rabbitmq.md | 129 + docs/dev/connectors/redis.md | 174 ++ docs/dev/connectors/twitter.md | 85 + docs/dev/datastream_api.md | 1779 +++++++++++++ docs/dev/event_time.md | 206 ++ docs/dev/event_timestamp_extractors.md | 106 + docs/dev/event_timestamps_watermarks.md | 329 +++ docs/dev/index.md | 25 + docs/dev/java8.md | 196 ++ docs/dev/libraries.md | 24 + docs/dev/libs/cep.md | 652 +++++ docs/dev/libs/gelly/graph_algorithms.md | 308 +++ docs/dev/libs/gelly/graph_api.md | 833 +++++++ docs/dev/libs/gelly/graph_generators.md | 654 +++++ docs/dev/libs/gelly/index.md | 69 + .../libs/gelly/iterative_graph_processing.md | 968 ++++++++ docs/dev/libs/gelly/library_methods.md | 347 +++ docs/dev/libs/ml/als.md | 175 ++ docs/dev/libs/ml/contribution_guide.md | 106 + docs/dev/libs/ml/cross_validation.md | 171 ++ docs/dev/libs/ml/distance_metrics.md | 107 + docs/dev/libs/ml/index.md | 144 ++ docs/dev/libs/ml/knn.md | 144 ++ docs/dev/libs/ml/min_max_scaler.md | 112 + docs/dev/libs/ml/multiple_linear_regression.md | 160 ++ docs/dev/libs/ml/optimization.md | 382 +++ docs/dev/libs/ml/pipelines.md | 441 ++++ docs/dev/libs/ml/polynomial_features.md | 108 + docs/dev/libs/ml/quickstart.md | 243 ++ docs/dev/libs/ml/standard_scaler.md | 113 + docs/dev/libs/ml/svm.md | 220 ++ docs/dev/libs/storm_compatibility.md | 287 +++ docs/dev/local_execution.md | 125 + docs/dev/quickstarts.md | 24 + docs/dev/scala_api_extensions.md | 408 +++ docs/dev/scala_shell.md | 193 ++ docs/dev/state.md | 293 +++ docs/dev/state_backends.md | 162 ++ docs/dev/table_api.md | 2079 ++++++++++++++++ docs/dev/types_serialization.md | 253 ++ docs/dev/windows.md | 677 +++++ docs/fig/ClientJmTm.svg | 348 +++ docs/fig/FlinkOnYarn.svg | 151 ++ docs/fig/back_pressure_sampling.png | Bin 0 -> 17635 bytes docs/fig/back_pressure_sampling_high.png | Bin 0 -> 77546 bytes docs/fig/back_pressure_sampling_in_progress.png | Bin 0 -> 79112 bytes docs/fig/back_pressure_sampling_ok.png | Bin 0 -> 79668 bytes docs/fig/checkpointing.svg | 1731 +++++++++++++ docs/fig/checkpoints.svg | 249 ++ docs/fig/event_ingestion_processing_time.svg | 375 +++ docs/fig/flink-on-emr.png | Bin 0 -> 103880 bytes docs/fig/gelly-example-graph.png | Bin 0 -> 18813 bytes docs/fig/gelly-filter.png | Bin 0 -> 57192 bytes docs/fig/gelly-gsa-sssp1.png | Bin 0 -> 31201 bytes docs/fig/gelly-reduceOnEdges.png | Bin 0 -> 23843 bytes docs/fig/gelly-reduceOnNeighbors.png | Bin 0 -> 34903 bytes docs/fig/gelly-union.png | Bin 0 -> 50498 bytes docs/fig/gelly-vc-sssp1.png | Bin 0 -> 28537 bytes docs/fig/iterations_delta_iterate_operator.png | Bin 0 -> 113607 bytes ...terations_delta_iterate_operator_example.png | Bin 0 -> 335057 bytes docs/fig/iterations_iterate_operator.png | Bin 0 -> 63465 bytes .../fig/iterations_iterate_operator_example.png | Bin 0 -> 102925 bytes docs/fig/iterations_supersteps.png | Bin 0 -> 54098 bytes docs/fig/job_and_execution_graph.svg | 851 +++++++ docs/fig/job_status.svg | 1049 ++++++++ docs/fig/jobmanager_ha_overview.png | Bin 0 -> 57875 bytes docs/fig/non-windowed.svg | 22 + docs/fig/parallel_dataflow.svg | 487 ++++ docs/fig/parallel_streams_watermarks.svg | 516 ++++ docs/fig/plan_visualizer.png | Bin 0 -> 145778 bytes docs/fig/processes.svg | 749 ++++++ docs/fig/program_dataflow.svg | 546 ++++ docs/fig/projects_dependencies.svg | 580 +++++ docs/fig/rescale.svg | 472 ++++ docs/fig/savepoints-overview.png | Bin 0 -> 62824 bytes docs/fig/savepoints-program_ids.png | Bin 0 -> 55492 bytes docs/fig/session-windows.svg | 22 + docs/fig/sliding-windows.svg | 22 + docs/fig/slot_sharing.svg | 721 ++++++ docs/fig/slots.svg | 505 ++++ docs/fig/slots_parallelism.svg | 695 ++++++ docs/fig/stack.svg | 606 +++++ docs/fig/state_machine.svg | 318 +++ docs/fig/state_partitioning.svg | 291 +++ docs/fig/stream_aligning.svg | 877 +++++++ docs/fig/stream_barriers.svg | 309 +++ docs/fig/stream_watermark_in_order.svg | 314 +++ docs/fig/stream_watermark_out_of_order.svg | 314 +++ docs/fig/tasks_chains.svg | 463 ++++ docs/fig/tasks_slots.svg | 395 +++ docs/fig/times_clocks.svg | 368 +++ docs/fig/tumbling-windows.svg | 22 + docs/fig/vertex-centric supersteps.png | Bin 0 -> 80198 bytes docs/fig/windows.svg | 193 ++ docs/index.md | 27 +- docs/internals/_draft_distributed_akka.md | 47 - docs/internals/add_operator.md | 24 +- docs/internals/back_pressure_monitoring.md | 83 - docs/internals/coding_guidelines.md | 25 - docs/internals/fig/ClientJmTm.svg | 348 --- docs/internals/fig/LICENSE.txt | 17 - docs/internals/fig/back_pressure_sampling.png | Bin 17635 -> 0 bytes .../fig/back_pressure_sampling_high.png | Bin 77546 -> 0 bytes .../fig/back_pressure_sampling_in_progress.png | Bin 79112 -> 0 bytes .../internals/fig/back_pressure_sampling_ok.png | Bin 79668 -> 0 bytes docs/internals/fig/checkpointing.svg | 1731 ------------- docs/internals/fig/job_and_execution_graph.svg | 851 ------- docs/internals/fig/job_status.svg | 1049 -------- docs/internals/fig/projects_dependencies.svg | 580 ----- docs/internals/fig/slots.svg | 505 ---- docs/internals/fig/stack.svg | 606 ----- docs/internals/fig/state_machine.svg | 318 --- docs/internals/fig/stream_aligning.svg | 877 ------- docs/internals/fig/stream_barriers.svg | 309 --- docs/internals/general_arch.md | 32 +- docs/internals/how_to_contribute.md | 25 - docs/internals/ide_setup.md | 19 +- docs/internals/index.md | 4 + docs/internals/job_scheduling.md | 23 +- docs/internals/logging.md | 106 - docs/internals/monitoring_rest_api.md | 589 ----- docs/internals/stream_checkpointing.md | 25 +- docs/internals/types_serialization.md | 258 -- docs/libs/cep/index.md | 25 - docs/libs/gelly_guide.md | 25 - docs/libs/index.md | 25 - docs/libs/ml/als.md | 25 - docs/libs/ml/contribution_guide.md | 25 - docs/libs/ml/distance_metrics.md | 25 - docs/libs/ml/index.md | 25 - docs/libs/ml/min_max_scaler.md | 25 - docs/libs/ml/multiple_linear_regression.md | 25 - docs/libs/ml/optimization.md | 25 - docs/libs/ml/pipelines.md | 25 - docs/libs/ml/polynomial_features.md | 25 - docs/libs/ml/quickstart.md | 25 - docs/libs/ml/standard_scaler.md | 25 - docs/libs/ml/svm.md | 25 - docs/libs/table.md | 29 - docs/monitoring/back_pressure.md | 81 + docs/monitoring/best_practices.md | 402 +++ docs/monitoring/index.md | 25 + docs/monitoring/logging.md | 98 + docs/monitoring/metrics.md | 468 ++++ docs/monitoring/rest_api.md | 586 +++++ docs/page/css/flink.css | 174 +- docs/page/font-awesome/css/font-awesome.css | 2199 ++++++++++++++++ docs/page/font-awesome/css/font-awesome.min.css | 4 + docs/page/font-awesome/fonts/FontAwesome.otf | Bin 0 -> 124988 bytes .../font-awesome/fonts/fontawesome-webfont.eot | Bin 0 -> 76518 bytes .../font-awesome/fonts/fontawesome-webfont.svg | 685 +++++ .../font-awesome/fonts/fontawesome-webfont.ttf | Bin 0 -> 152796 bytes .../font-awesome/fonts/fontawesome-webfont.woff | Bin 0 -> 90412 bytes .../fonts/fontawesome-webfont.woff2 | Bin 0 -> 71896 bytes docs/quickstart/java_api_quickstart.md | 22 +- docs/quickstart/run_example_quickstart.md | 7 +- docs/quickstart/scala_api_quickstart.md | 21 +- docs/quickstart/setup_quickstart.md | 11 +- docs/redirects/back_pressure.md | 24 + docs/redirects/basic_api_concepts.md | 24 + docs/redirects/best_practices.md | 24 + docs/redirects/cassandra.md | 24 + docs/redirects/cep.md | 24 + docs/redirects/cli.md | 24 + docs/redirects/cluster_execution.md | 24 + docs/redirects/concepts.md | 24 + docs/redirects/connectors.md | 24 + docs/redirects/datastream_api.md | 24 + docs/redirects/elasticsearch.md | 24 + docs/redirects/elasticsearch2.md | 24 + docs/redirects/event_time.md | 24 + docs/redirects/event_timestamp_extractors.md | 24 + docs/redirects/event_timestamps_watermarks.md | 24 + docs/redirects/fault_tolerance.md | 24 + docs/redirects/filesystem_sink.md | 24 + docs/redirects/gelly.md | 24 + docs/redirects/java8.md | 24 + docs/redirects/kafka.md | 24 + docs/redirects/kinesis.md | 24 + docs/redirects/local_execution.md | 24 + docs/redirects/metrics.md | 24 + docs/redirects/ml.md | 24 + docs/redirects/programming_guide.md | 24 + docs/redirects/rest_api.md | 24 + docs/redirects/savepoints.md | 24 + docs/redirects/scala_api_extensions.md | 24 + docs/redirects/scala_shell.md | 24 + docs/redirects/state.md | 24 + docs/redirects/state_backends.md | 24 + docs/redirects/storm_compat.md | 24 + docs/redirects/streaming_guide.md | 24 + docs/redirects/table.md | 24 + docs/redirects/types_serialization.md | 24 + docs/redirects/windows.md | 24 + docs/setup/aws.md | 12 +- docs/setup/building.md | 5 +- docs/setup/cli.md | 322 +++ docs/setup/cluster_setup.md | 7 +- docs/setup/config.md | 12 +- docs/setup/deployment.md | 24 + docs/setup/fault_tolerance.md | 460 ++++ docs/setup/fig/FlinkOnYarn.svg | 151 -- docs/setup/fig/LICENSE.txt | 17 - docs/setup/fig/flink-on-emr.png | Bin 103880 -> 0 bytes docs/setup/fig/jobmanager_ha_overview.png | Bin 57875 -> 0 bytes docs/setup/fig/slots_parallelism.svg | 695 ------ docs/setup/gce_setup.md | 6 +- docs/setup/index.md | 6 +- docs/setup/jobmanager_high_availability.md | 12 +- docs/setup/local_setup.md | 7 +- docs/setup/savepoints.md | 109 + docs/setup/yarn_setup.md | 12 +- pom.xml | 1 + 350 files changed, 45243 insertions(+), 42334 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/844c874b/LICENSE ---------------------------------------------------------------------- diff --git a/LICENSE b/LICENSE index 07df05f..f7699da 100644 --- a/LICENSE +++ b/LICENSE @@ -225,7 +225,7 @@ The Apache Flink project bundles the following files under the MIT License: - dagre v0.7.4 (https://github.com/cpettitt/dagre) - Copyright (c) 2012-2014 Chris Pettitt - dagre-d3 v0.4.17 (https://github.com/cpettitt/dagre-d3) - Copyright (c) 2013 Chris Pettitt - EvEmitter v1.0.2 (https://github.com/metafizzy/ev-emitter) - Copyright (C) 2016 David DeSandro - - Font Awesome (code) v4.5.0 (http://fontawesome.io) - Copyright (c) 2014 Dave Gandy + - Font Awesome (code) v4.5.0, v4.6.3 (http://fontawesome.io) - Copyright (c) 2014 Dave Gandy - graphlib v1.0.7 (https://github.com/cpettitt/graphlib) - Copyright (c) 2012-2014 Chris Pettitt - imagesloaded v4.1.0 (https://github.com/desandro/imagesloaded) - Copyright (C) 2016 David DeSandro - JQuery v2.2.0 (http://jquery.com/) - Copyright 2014 jQuery Foundation and other contributors @@ -300,7 +300,8 @@ The Apache Flink project bundles the following fonts under the Open Font License (OFT) - http://scripts.sil.org/OFL - Font Awesome (http://fortawesome.github.io/Font-Awesome/) - Created by Dave Gandy - -> fonts in "flink-runtime-web/web-dashboard/assets/fonts" + -> fonts in "flink-runtime-web/web-dashboard/web/fonts" + -> fonts in "docs/page/font-awesome/fonts" ----------------------------------------------------------------------- The ISC License http://git-wip-us.apache.org/repos/asf/flink/blob/844c874b/docs/README.md ---------------------------------------------------------------------- diff --git a/docs/README.md b/docs/README.md index 52dfad3..879c33b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -109,43 +109,19 @@ These will be replaced by a info or warning label. You can change the text of th ### Documentation -#### Top Navigation +#### Navigation -You can modify the top-level navigation in two places. You can either edit the `_includes/navbar.html` file or add tags to your page frontmatter (recommended). +The navigation on the left side of the docs is automatically generated when building the docs. You can modify the markup in `_include/sidenav.html`. - # Top-level navigation - top-nav-group: apis - top-nav-pos: 2 - top-nav-title: <strong>Batch Guide</strong> (DataSet API) +The structure of the navigation is determined by the front matter of all pages. The fields used to determine the structure are: -This adds the page to the group `apis` (via `top-nav-group`) at position `2` (via `top-nav-pos`). Furthermore, it specifies a custom title for the navigation via `top-nav-title`. If this field is missing, the regular page title (via `title`) will be used. If no position is specified, the element will be added to the end of the group. If no group is specified, the page will not show up. +- `nav-id` => ID of this page. Other pages can use this ID as their parent ID. +- `nav-parent_id` => ID of the parent. This page will be listed under the page with id `nav-parent_id`. -Currently, there are groups `quickstart`, `setup`, `deployment`, `apis`, `libs`, and `internals`. +Level 0 is made up of all pages, which have nav-parent_id set to `root`. There is no limitation on how many levels you can nest. -#### Sub Navigation +The `title` of the page is used as the default link text. You can override this via `nav-title`. The relative position per navigational level is determined by `nav-pos`. -A sub navigation is shown if the field `sub-nav-group` is specified. A sub navigation groups all pages with the same `sub-nav-group`. Check out the streaming or batch guide as an example. +If you have a page with sub pages, the link target will be used to expand the sub level navigation. If you want to actually add a link to the page as well, you can add the `nav-show_overview: true` field to the front matter. This will then add an `Overview` sub page to the expanded list. - # Sub-level navigation - sub-nav-group: batch - sub-nav-id: dataset_api - sub-nav-pos: 1 - sub-nav-title: DataSet API - -The fields work similar to their `top-nav-*` counterparts. - -In addition, you can specify a hierarchy via `sub-nav-id` and `sub-nav-parent`: - - # Sub-level navigation - sub-nav-group: batch - sub-nav-parent: dataset_api - sub-nav-pos: 1 - sub-nav-title: Transformations - -This will show the `Transformations` page under the `DataSet API` page. The `sub-nav-parent` field has to have a matching `sub-nav-id`. - -#### Breadcrumbs - -Pages with sub navigations can use breadcrumbs like `Batch Guide > Libraries > Machine Learning > Optimization`. - -The breadcrumbs for the last page are generated from the front matter. For the a sub navigation root to appear (like `Batch Guide` in the example above), you have to specify the `sub-nav-group-title`. This field designates a group page as the root. +The nesting is also used for the breadcrumbs like `Application Development > Libraries > Machine Learning > Optimization`. http://git-wip-us.apache.org/repos/asf/flink/blob/844c874b/docs/_config.yml ---------------------------------------------------------------------- diff --git a/docs/_config.yml b/docs/_config.yml index d9bb57e..700d289 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -29,6 +29,7 @@ version: "1.2-SNAPSHOT" version_hadoop1: "1.2-hadoop1-SNAPSHOT" version_short: "1.2" # Used for the top navbar w/o snapshot suffix +is_snapshot_version: true # This suffix is appended to the Scala-dependent Maven artifact names scala_version_suffix: "_2.10" @@ -40,6 +41,16 @@ jira_url: "https://issues.apache.org/jira/browse/FLINK"; github_url: "https://github.com/apache/flink"; download_url: "http://flink.apache.org/downloads.html"; +# Flag whether this is the latest stable version or not. If not, a warning +# will be printed pointing to the docs of the latest stable version. +is_latest: true +is_stable: false +latest_stable_url: http://ci.apache.org/projects/flink/flink-docs-release-1.1 + +previous_docs: + 1.1: http://ci.apache.org/projects/flink/flink-docs-release-1.1 + 1.0: http://ci.apache.org/projects/flink/flink-docs-release-1.0 + #------------------------------------------------------------------------------ # BUILD CONFIG #------------------------------------------------------------------------------ @@ -47,14 +58,16 @@ download_url: "http://flink.apache.org/downloads.html"; # to change anything here. #------------------------------------------------------------------------------ +# Used in some documents to initialize arrays. Don't delete. +array: [] + defaults: - scope: path: "" values: layout: plain - top-nav-pos: 99999 # Move to end - sub-nav-pos: 99999 # Move to end + nav-pos: 99999 # Move to end if no pos specified markdown: KramdownPygments highlighter: pygments http://git-wip-us.apache.org/repos/asf/flink/blob/844c874b/docs/_includes/navbar.html ---------------------------------------------------------------------- diff --git a/docs/_includes/navbar.html b/docs/_includes/navbar.html deleted file mode 100644 index 5821a46..0000000 --- a/docs/_includes/navbar.html +++ /dev/null @@ -1,117 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -{% capture quickstart %}{{site.baseurl}}/quickstart{% endcapture %} -{% capture setup %}{{site.baseurl}}/setup{% endcapture %} -{% capture apis %}{{site.baseurl}}/apis{% endcapture %} -{% capture libs %}{{site.baseurl}}/libs{% endcapture %} -{% capture internals %}{{site.baseurl}}/internals{% endcapture %} - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="{{ site.baseurl }}/page/img/navbar-brand-logo.jpg"></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li class="hidden-sm {% if page.url == '/' %}active{% endif %}"><a href="{{ site.baseurl}}/">Docs v{{ site.version_short }}</a></li> - - <li class="{% if page.url == '/concepts/concepts.html' %}active{% endif %}"><a href="{{ site.baseurl}}/concepts/concepts.html">Concepts</a></li> - - <!-- Setup --> - <li class="dropdown{% if page.url contains '/setup/' %} active{% endif %}"> - <a href="{{ setup }}" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - {% assign setup_group = (site.pages | where: "top-nav-group" , "setup" | sort: "top-nav-pos") %} - {% for setup_group_page in setup_group %} - <li class="{% if page.url contains setup_group_page.url %}active{% endif %}"><a href="{{ site.baseurl }}{{ setup_group_page.url }}">{% if setup_group_page.top-nav-title %}{{ setup_group_page.top-nav-title }}{% else %}{{ setup_group_page.title }}{% endif %}</a></li> - {% endfor %} - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Quickstart</strong></li> - - <!-- Quickstart --> - {% assign quickstart_group = (site.pages | where: "top-nav-group" , "quickstart" | sort: "top-nav-pos") %} - {% for quickstart_page in quickstart_group %} - <li class="{% if page.url contains quickstart_page.url %}active{% endif %}"><a href="{{ site.baseurl }}{{ quickstart_page.url }}">{% if quickstart_page.top-nav-title %}{{ quickstart_page.top-nav-title }}{% else %}{{ quickstart_page.title }}{% endif %}</a></li> - {% endfor %} - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - {% assign deployment_group = (site.pages | where: "top-nav-group" , "deployment" | sort: "top-nav-pos") %} - {% for deployment_group_page in deployment_group %} - <li class="{% if page.url contains deployment_group_page.url %}active{% endif %}"><a href="{{ site.baseurl }}{{ deployment_group_page.url }}">{% if deployment_group_page.top-nav-title %}{{ deployment_group_page.top-nav-title }}{% else %}{{ deployment_group_page.title }}{% endif %}</a></li> - {% endfor %} - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown{% unless page.url contains '/libs/' %}{% if page.url contains '/apis/' %} active{% endif %}{% endunless %}"> - <a href="{{ apis }}" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - {% assign apis_group = (site.pages | where: "top-nav-group" , "apis" | sort: "top-nav-pos") %} - {% for apis_group_page in apis_group %} - <li class="{% if page.url contains apis_group_page.url %}active{% endif %}"><a href="{{ site.baseurl }}{{ apis_group_page.url }}">{% if apis_group_page.top-nav-title %}{{ apis_group_page.top-nav-title }}{% else %}{{ apis_group_page.title }}{% endif %}</a></li> - {% endfor %} - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown{% if page.url contains '/libs/' %} active{% endif %}"> - <a href="{{ libs }}" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - {% assign libs_group = (site.pages | where: "top-nav-group" , "libs" | sort: "top-nav-pos") %} - {% for libs_page in libs_group %} - <li class="{% if page.url contains libs_page.url %}active{% endif %}"><a href="{{ site.baseurl }}{{ libs_page.url }}">{% if libs_page.top-nav-title %}{{ libs_page.top-nav-title }}{% else %}{{ libs_page.title }}{% endif %}</a></li> - {% endfor %} - </ul> - </li> - - <!-- Internals --> - <li class="dropdown{% if page.url contains '/internals/' %} active{% endif %}"> - <a href="{{ internals }}" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/how-to-contribute.html";><small><span class="glyphicon glyphicon-new-window"></span></small> How to Contribute</a></li> - <li><a href="http://flink.apache.org/contribute-code.html#coding-guidelines";><small><span class="glyphicon glyphicon-new-window"></span></small> Coding Guidelines</a></li> - {% assign internals_group = (site.pages | where: "top-nav-group" , "internals" | sort: "top-nav-pos") %} - {% for internals_page in internals_group %} - <li class="{% if page.url contains internals_page.url %}active{% endif %}"><a href="{{ site.baseurl }}{{ internals_page.url }}">{% if internals_page.top-nav-title %}{{ internals_page.top-nav-title }}{% else %}{{ internals_page.title }}{% endif %}</a></li> - {% endfor %} - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="{{site.baseurl}}/search-results.html"> - <div class="form-group"> - <input type="text" class="form-control" size="16px" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> http://git-wip-us.apache.org/repos/asf/flink/blob/844c874b/docs/_includes/sidenav.html ---------------------------------------------------------------------- diff --git a/docs/_includes/sidenav.html b/docs/_includes/sidenav.html new file mode 100644 index 0000000..b56bcf2 --- /dev/null +++ b/docs/_includes/sidenav.html @@ -0,0 +1,149 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +{% comment %} +============================================================================== +Extract the active nav IDs. +============================================================================== +{% endcomment %} + +{% assign active_nav_ids = site.array %} +{% assign parent_id = page.nav-parent_id %} + +{% for i in (1..10) %} + {% if parent_id %} + {% assign active_nav_ids = active_nav_ids | push: parent_id %} + {% assign current = (site.pages | where: "nav-id" , parent_id | sort: "nav-pos") %} + {% if current.size > 0 %} + {% assign parent_id = current[0].nav-parent_id %} + {% else %} + {% break %} + {% endif %} + {% else %} + {% break %} + {% endif %} +{% endfor %} + +{% comment %} +============================================================================== +Build the nested list from nav-id and nav-parent_id relations. +============================================================================== +This builds a nested list from all pages. The fields used to determine the +structure are: + +- 'nav-id' => ID of this page. Other pages can use this ID as their + parent ID. +- 'nav-parent_id' => ID of the parent. This page will be listed under + the page with id 'nav-parent_id'. + +Level 0 is made up of all pages, which have nav-parent_id set to 'root'. + +The 'title' of the page is used as the default link text. You can +override this via 'nav-title'. The relative position per navigational +level is determined by 'nav-pos'. +{% endcomment %} + +{% assign elementsPosStack = site.array %} +{% assign posStack = site.array %} + +{% assign elements = site.array %} +{% assign children = (site.pages | where: "nav-parent_id" , "root" | sort: "nav-pos") %} +{% if children.size > 0 %} + {% assign elements = elements | push: children %} +{% endif %} + +{% assign elementsPos = 0 %} +{% assign pos = 0 %} + +<div class="sidenav-logo"> + <p><a href="{{ site.baseurl }}"><img class="bottom" alt="Apache Flink" src="{{ site.baseurl }}/page/img/navbar-brand-logo.jpg"></a> v{{ site.version }}</p> +</div> +<ul id="sidenav"> +{% for i in (1..10000) %} + {% if pos >= elements[elementsPos].size %} + {% if elementsPos == 0 %} + {% break %} + {% else %} + {% assign elementsPos = elementsPosStack | last %} + {% assign pos = posStack | last %} +</li></ul></div> + {% assign elementsPosStack = elementsPosStack | pop %} + {% assign posStack = posStack | pop %} + {% endif %} + {% else %} + {% assign this = elements[elementsPos][pos] %} + + {% if this.url == page.url %} + {% assign active = true %} + {% elsif this.nav-id and active_nav_ids contains this.nav-id %} + {% assign active = true %} + {% else %} + {% assign active = false %} + {% endif %} + + {% capture title %}{% if this.nav-title %}{{ this.nav-title }}{% else %}{{ this.title }}{% endif %}{% endcapture %} + {% capture target %}"{{ site.baseurl }}{{ this.url }}"{% if active %} class="active"{% endif %}{% endcapture %} + {% capture overview_target %}"{{ site.baseurl }}{{ this.url }}"{% if this.url == page.url %} class="active"{% endif %}{% endcapture %} + + {% assign pos = pos | plus: 1 %} + {% if this.nav-id %} + {% assign children = (site.pages | where: "nav-parent_id" , this.nav-id | sort: "nav-pos") %} + {% if children.size > 0 %} + {% capture collapse_target %}"#collapse-{{ i }}" data-toggle="collapse"{% if active %} class="active"{% endif %}{% endcapture %} + {% capture expand %}{% unless active %} <i class="fa fa-caret-down pull-right" aria-hidden="true" style="padding-top: 4px"></i>{% endunless %}{% endcapture %} +<li><a href={{ collapse_target }}>{{ title }}{{ expand }}</a><div class="collapse{% if active %} in{% endif %}" id="collapse-{{ i }}"><ul> + {% if this.nav-show_overview %}<li><a href={{ overview_target }}>Overview</a></li>{% endif %} + {% assign elements = elements | push: children %} + {% assign elementsPosStack = elementsPosStack | push: elementsPos %} + {% assign posStack = posStack | push: pos %} + + {% assign elementsPos = elements.size | minus: 1 %} + {% assign pos = 0 %} + {% else %} +<li><a href={{ target }}>{{ title }}</a></li> + {% endif %} + {% else %} +<li><a href={{ target }}>{{ title }}</a></li> + {% endif %} + {% endif %} +{% endfor %} + <li class="divider"></li> + <li><a href="http://flink.apache.org";><i class="fa fa-external-link" aria-hidden="true"></i> Project Page</a></li> +</ul> + +<div class="sidenav-search-box"> + <form class="navbar-form" role="search" action="{{site.baseurl}}/search-results.html"> + <div class="form-group"> + <input type="text" class="form-control" size="16px" name="q" placeholder="Search Docs"> + </div> + <button type="submit" class="btn btn-default">Go</button> + </form> +</div> + +<div class="sidenav-versions"> + <div class="dropdown"> + <button class="btn btn-default dropdown-toggle" type="button" data-toggle="dropdown">Pick Docs Version + <span class="caret"></span></button> + <ul class="dropdown-menu"> + {% for d in site.previous_docs %} + <li><a href="{{ d[1] }}">v{{ d[0] }}</a></li> + {% endfor %} + </ul> + </div> +</div> http://git-wip-us.apache.org/repos/asf/flink/blob/844c874b/docs/_layouts/base.html ---------------------------------------------------------------------- diff --git a/docs/_layouts/base.html b/docs/_layouts/base.html index 690d2af..21065ec 100644 --- a/docs/_layouts/base.html +++ b/docs/_layouts/base.html @@ -32,6 +32,7 @@ under the License. <link rel="stylesheet" href="{{ site.baseurl }}/page/css/flink.css"> <link rel="stylesheet" href="{{ site.baseurl }}/page/css/syntax.css"> <link rel="stylesheet" href="{{ site.baseurl }}/page/css/codetabs.css"> + <link rel="stylesheet" href="{{ site.baseurl }}/page/font-awesome/css/font-awesome.min.css"> {% if page.mathjax %} <script type="text/x-mathjax-config"> MathJax.Hub.Config({ @@ -53,20 +54,36 @@ under the License. <![endif]--> </head> <body> - {% comment %} Includes are found in the _includes directory. {% endcomment %} - {% include navbar.html %} - - {% if page.mathjax %} - {% include latex_commands.html %} - {% endif %} - <!-- Main content. --> <div class="container"> + {% if site.is_stable %} + {% unless site.is_latest %} + <div class="row"> + <div class="col-sm-12"> + <div class="alert alert-info"> + <strong>Note</strong>: This documentation is for Flink version <strong>{{ site.version }}</strong>. There is a more recent stable version available. Please consider updating and <a href="{{ site.latest_stable_url }}">check out the documentation for that version</a>. + </div> + </div> + </div> + {% endunless %} + {% endif %} + {% comment %} This is the base for all content. The content from the layouts found in the _layouts directory goes here. {% endcomment %} - {{ content }} + <div class="row"> + <div class="col-lg-3"> + {% include sidenav.html %} + </div> + <div class="col-lg-9 content"> + {% if page.mathjax %} + {% include latex_commands.html %} + {% endif %} + + {{ content }} + </div> + </div> </div><!-- /.container --> <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> http://git-wip-us.apache.org/repos/asf/flink/blob/844c874b/docs/_layouts/plain.html ---------------------------------------------------------------------- diff --git a/docs/_layouts/plain.html b/docs/_layouts/plain.html index 7a076a2..63a6681 100644 --- a/docs/_layouts/plain.html +++ b/docs/_layouts/plain.html @@ -19,103 +19,39 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> -<div class="row"> -{% if page.sub-nav-group %} -{% comment %} -The plain layout with a sub navigation. -- This is activated via the 'sub-nav-group' field in the preemble. -- All pages of this sub nav group will be displayed in the sub navigation: - * Each element without a 'sub-nav-parent' field will be displayed on the 1st level, where the position is defined via 'sub-nav-pos'. - * If the page should be displayed as a child element, it needs to specify a 'sub-nav-parent' field, which matches the 'sub-nav-id' of its parent. The parent only needs to specify this if it expects child nodes. -{% endcomment %} - <!-- Sub Navigation --> - <div class="col-sm-3"> - <ul id="sub-nav"> - {% comment %} Get all pages belonging to this group sorted by their position {% endcomment %} - {% assign group = (site.pages | where: "sub-nav-group" , page.sub-nav-group | where: "sub-nav-parent" , nil | sort: "sub-nav-pos") %} - {% for group_page in group %} - {% if group_page.sub-nav-id %} - {% assign sub_group = (site.pages | where: "sub-nav-group" , page.sub-nav-group | where: "sub-nav-parent" , group_page.sub-nav-id | sort: "sub-nav-pos") %} - {% else %} - {% assign sub_group = nil %} - {% endif %} - <li><a href="{{ site.baseurl }}{{ group_page.url }}" class="{% if page.url contains group_page.url %}active{% endif %}">{% if group_page.sub-nav-title %}{{ group_page.sub-nav-title }}{% else %}{{ group_page.title }}{% endif %}</a> - {% if sub_group and sub_group.size() > 0 %} - <ul> - {% for sub_group_page in sub_group %} - <li><a href="{{ site.baseurl }}{{ sub_group_page.url }}" class="{% if page.url contains sub_group_page.url or (sub_group_page.sub-nav-id and page.sub-nav-parent and sub_group_page.sub-nav-id == page.sub-nav-parent) %}active{% endif %}">{% if sub_group_page.sub-nav-title %}{{ sub_group_page.sub-nav-title }}{% else %}{{ sub_group_page.title }}{% endif %}</a></li> - {% endfor %} - </ul> - {% endif %} - </li> - {% endfor %} - </ul> - </div> - <!-- Main --> - <div class="col-sm-9"> - <!-- Top anchor --> - <a href="#top"></a> +{% assign active_pages = site.array %} +{% assign active = page %} + +{% for i in (1..10) %} + {% assign active_pages = active_pages | push: active %} + {% if active.nav-parent_id %} + {% assign next = (site.pages | where: "nav-id" , active.nav-parent_id ) %} + {% if next.size > 0 %} + {% assign active = next[0] %} + {% else %} + {% break %} + {% endif %} + {% else %} + {% break %} + {% endif %} +{% endfor %} + +{% assign active_pages = active_pages | reverse %} + +<ol class="breadcrumb"> +{% for p in active_pages %} + {% capture title %}{% if p.nav-title %}{{ p.nav-title }}{% else %}{{ p.title }}{% endif %}{% endcapture %} + {% if forloop.last == true %} + <li class="active">{{ title }}</li> + {% elsif p.nav-show_overview %} + <li><a href="{{ site.baseurl }}{{ p.url }}">{{ title }}</a></li> + {% else %} + <li>{{ title }}</li> + {% endif %} +{% endfor %} +</ol> + +<h1>{{ page.title }}{% if page.is_beta %} <span class="beta">Beta</span>{% endif %}</h1> - <!-- Artifact name change warning. Remove for the 1.0 release. --> - <div class="panel panel-default"> - <div class="panel-body"><strong>Important</strong>: Maven artifacts which depend on Scala are now suffixed with the Scala major version, e.g. "2.10" or "2.11". Please consult the <a href="https://cwiki.apache.org/confluence/display/FLINK/Maven+artifact+names+suffixed+with+Scala+version";>migration guide on the project Wiki</a>.</div> - </div> - - <!-- Breadcrumbs above the main heading --> - <ol class="breadcrumb"> - {% for group_page in group %} - {% if group_page.sub-nav-group-title %} - <li><a href="{{ site.baseurl }}{{ group_page.url }}">{{ group_page.sub-nav-group-title }}</a></li> - {% endif %} - {% endfor %} - - {% if page.sub-nav-parent %} - {% assign parent = (site.pages | where: "sub-nav-group" , page.sub-nav-group | where: "sub-nav-id" , page.sub-nav-parent | first) %} - {% if parent %} - - {% if parent.sub-nav-parent %} - {% assign grandparent = (site.pages | where: "sub-nav-group" , page.sub-nav-group | where: "sub-nav-id" , parent.sub-nav-parent | first) %} - - {% if grandparent %} - <li><a href="{{ site.baseurl }}{{ grandparent.url }}">{% if grandparent.sub-nav-title %}{{ grandparent.sub-nav-title }}{% else %}{{ grandparent.title }}{% endif %}</a></li> - {% endif %} - - {% endif %} - - <li><a href="{{ site.baseurl }}{{ parent.url }}">{% if parent.sub-nav-title %}{{ parent.sub-nav-title }}{% else %}{{ parent.title }}{% endif %}</a></li> - {% endif %} - {% endif %} - <li class="active">{% if page.sub-nav-title %}{{ page.sub-nav-title }}{% else %}{{ page.title }}{% endif %}</li> - </ol> - - <div class="text"> - <!-- Main heading --> - <h1>{{ page.title }}{% if page.is_beta %} <span class="beta">(Beta)</span>{% endif %}</h1> - - <!-- Content --> - {{ content }} - </div> - </div> -{% else %} -{% comment %} -The plain layout without a sub navigation (only text). -{% endcomment %} - <div class="col-md-8 col-md-offset-2 text"> - <!-- Artifact name change warning. Remove for the 1.0 release. --> - <div class="panel panel-default"> - <div class="panel-body"><strong>Important</strong>: Maven artifacts which depend on Scala are now suffixed with the Scala major version, e.g. "2.10" or "2.11". Please consult the <a href="https://cwiki.apache.org/confluence/display/FLINK/Maven+artifact+names+suffixed+with+Scala+version";>migration guide on the project Wiki</a>.</div> - </div> - - <h1>{{ page.title }}{% if page.is_beta %} <span class="beta">Beta</span>{% endif %}</h1> {{ content }} - </div> -{% endif %} - {% comment %} - Removed until Robert complains... ;) - <div class="col-sm-8 col-sm-offset-2"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> - {% endcomment %} -</div> http://git-wip-us.apache.org/repos/asf/flink/blob/844c874b/docs/_layouts/redirect.html ---------------------------------------------------------------------- diff --git a/docs/_layouts/redirect.html b/docs/_layouts/redirect.html new file mode 100644 index 0000000..ff3d70c --- /dev/null +++ b/docs/_layouts/redirect.html @@ -0,0 +1,27 @@ +--- +layout: base +--- +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<meta http-equiv="refresh" content="1; url={{ site.baseurl }}{{ page.redirect }}" /> + +<h1>Page '{{ page.title }}' Has Moved</h1> + +The <strong>{{ page.title }}</strong> has been moved. Redirecting to <a href="{{ site.baseurl }}/{{ page.redirect }}">{{ site.baseurl }}{{ page.redirect }}</a> in 1 second. http://git-wip-us.apache.org/repos/asf/flink/blob/844c874b/docs/apis/batch/connectors.md ---------------------------------------------------------------------- diff --git a/docs/apis/batch/connectors.md b/docs/apis/batch/connectors.md deleted file mode 100644 index 21ed260..0000000 --- a/docs/apis/batch/connectors.md +++ /dev/null @@ -1,242 +0,0 @@ ---- -title: "Connectors" - -# Sub-level navigation -sub-nav-group: batch -sub-nav-pos: 4 ---- -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> - -* TOC -{:toc} - -## Reading from file systems - -Flink has build-in support for the following file systems: - -| Filesystem | Scheme | Notes | -| ------------------------------------- |--------------| ------ | -| Hadoop Distributed File System (HDFS) | `hdfs://` | All HDFS versions are supported | -| Amazon S3 | `s3://` | Support through Hadoop file system implementation (see below) | -| MapR file system | `maprfs://` | The user has to manually place the required jar files in the `lib/` dir | -| Alluxio | `alluxio://` | Support through Hadoop file system implementation (see below) | - - - -### Using Hadoop file system implementations - -Apache Flink allows users to use any file system implementing the `org.apache.hadoop.fs.FileSystem` -interface. There are Hadoop `FileSystem` implementations for - -- [S3](https://aws.amazon.com/s3/) (tested) -- [Google Cloud Storage Connector for Hadoop](https://cloud.google.com/hadoop/google-cloud-storage-connector) (tested) -- [Alluxio](http://alluxio.org/) (tested) -- [XtreemFS](http://www.xtreemfs.org/) (tested) -- FTP via [Hftp](http://hadoop.apache.org/docs/r1.2.1/hftp.html) (not tested) -- and many more. - -In order to use a Hadoop file system with Flink, make sure that - -- the `flink-conf.yaml` has set the `fs.hdfs.hadoopconf` property set to the Hadoop configuration directory. -- the Hadoop configuration (in that directory) has an entry for the required file system. Examples for S3 and Alluxio are shown below. -- the required classes for using the file system are available in the `lib/` folder of the Flink installation (on all machines running Flink). If putting the files into the directory is not possible, Flink is also respecting the `HADOOP_CLASSPATH` environment variable to add Hadoop jar files to the classpath. - -#### Amazon S3 - -For Amazon S3 support add the following entries into the `core-site.xml` file: - -~~~xml -<!-- configure the file system implementation --> -<property> - <name>fs.s3.impl</name> - <value>org.apache.hadoop.fs.s3native.NativeS3FileSystem</value> -</property> - -<!-- set your AWS ID --> -<property> - <name>fs.s3.awsAccessKeyId</name> - <value>putKeyHere</value> -</property> - -<!-- set your AWS access key --> -<property> - <name>fs.s3.awsSecretAccessKey</name> - <value>putSecretHere</value> -</property> -~~~ - -#### Alluxio - -For Alluxio support add the following entry into the `core-site.xml` file: - -~~~xml -<property> - <name>fs.alluxio.impl</name> - <value>alluxio.hadoop.FileSystem</value> -</property> -~~~ - - -## Connecting to other systems using Input/OutputFormat wrappers for Hadoop - -Apache Flink allows users to access many different systems as data sources or sinks. -The system is designed for very easy extensibility. Similar to Apache Hadoop, Flink has the concept -of so called `InputFormat`s and `OutputFormat`s. - -One implementation of these `InputFormat`s is the `HadoopInputFormat`. This is a wrapper that allows -users to use all existing Hadoop input formats with Flink. - -This section shows some examples for connecting Flink to other systems. -[Read more about Hadoop compatibility in Flink]({{ site.baseurl }}/apis/batch/hadoop_compatibility.html). - -## Avro support in Flink - -Flink has extensive build-in support for [Apache Avro](http://avro.apache.org/). This allows to easily read from Avro files with Flink. -Also, the serialization framework of Flink is able to handle classes generated from Avro schemas. - -In order to read data from an Avro file, you have to specify an `AvroInputFormat`. - -**Example**: - -~~~java -AvroInputFormat<User> users = new AvroInputFormat<User>(in, User.class); -DataSet<User> usersDS = env.createInput(users); -~~~ - -Note that `User` is a POJO generated by Avro. Flink also allows to perform string-based key selection of these POJOs. For example: - -~~~java -usersDS.groupBy("name") -~~~ - - -Note that using the `GenericData.Record` type is possible with Flink, but not recommended. Since the record contains the full schema, its very data intensive and thus probably slow to use. - -Flink's POJO field selection also works with POJOs generated from Avro. However, the usage is only possible if the field types are written correctly to the generated class. If a field is of type `Object` you can not use the field as a join or grouping key. -Specifying a field in Avro like this `{"name": "type_double_test", "type": "double"},` works fine, however specifying it as a UNION-type with only one field (`{"name": "type_double_test", "type": ["double"]},`) will generate a field of type `Object`. Note that specifying nullable types (`{"name": "type_double_test", "type": ["null", "double"]},`) is possible! - - - -### Access Microsoft Azure Table Storage - -_Note: This example works starting from Flink 0.6-incubating_ - -This example is using the `HadoopInputFormat` wrapper to use an existing Hadoop input format implementation for accessing [Azure's Table Storage](https://azure.microsoft.com/en-us/documentation/articles/storage-introduction/). - -1. Download and compile the `azure-tables-hadoop` project. The input format developed by the project is not yet available in Maven Central, therefore, we have to build the project ourselves. -Execute the following commands: - - ~~~bash - git clone https://github.com/mooso/azure-tables-hadoop.git - cd azure-tables-hadoop - mvn clean install - ~~~ - -2. Setup a new Flink project using the quickstarts: - - ~~~bash - curl https://flink.apache.org/q/quickstart.sh | bash - ~~~ - -3. Add the following dependencies (in the `<dependencies>` section) to your `pom.xml` file: - - ~~~xml - <dependency> - <groupId>org.apache.flink</groupId> - <artifactId>flink-hadoop-compatibility{{ site.scala_version_suffix }}</artifactId> - <version>{{site.version}}</version> - </dependency> - <dependency> - <groupId>com.microsoft.hadoop</groupId> - <artifactId>microsoft-hadoop-azure</artifactId> - <version>0.0.4</version> - </dependency> - ~~~ - - `flink-hadoop-compatibility` is a Flink package that provides the Hadoop input format wrappers. - `microsoft-hadoop-azure` is adding the project we've build before to our project. - -The project is now prepared for starting to code. We recommend to import the project into an IDE, such as Eclipse or IntelliJ. (Import as a Maven project!). -Browse to the code of the `Job.java` file. Its an empty skeleton for a Flink job. - -Paste the following code into it: - -~~~java -import java.util.Map; -import org.apache.flink.api.common.functions.MapFunction; -import org.apache.flink.api.java.DataSet; -import org.apache.flink.api.java.ExecutionEnvironment; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.hadoopcompatibility.mapreduce.HadoopInputFormat; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.Job; -import com.microsoft.hadoop.azure.AzureTableConfiguration; -import com.microsoft.hadoop.azure.AzureTableInputFormat; -import com.microsoft.hadoop.azure.WritableEntity; -import com.microsoft.windowsazure.storage.table.EntityProperty; - -public class AzureTableExample { - - public static void main(String[] args) throws Exception { - // set up the execution environment - final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); - - // create a AzureTableInputFormat, using a Hadoop input format wrapper - HadoopInputFormat<Text, WritableEntity> hdIf = new HadoopInputFormat<Text, WritableEntity>(new AzureTableInputFormat(), Text.class, WritableEntity.class, new Job()); - - // set the Account URI, something like: https://apacheflink.table.core.windows.net - hdIf.getConfiguration().set(AzureTableConfiguration.Keys.ACCOUNT_URI.getKey(), "TODO"); - // set the secret storage key here - hdIf.getConfiguration().set(AzureTableConfiguration.Keys.STORAGE_KEY.getKey(), "TODO"); - // set the table name here - hdIf.getConfiguration().set(AzureTableConfiguration.Keys.TABLE_NAME.getKey(), "TODO"); - - DataSet<Tuple2<Text, WritableEntity>> input = env.createInput(hdIf); - // a little example how to use the data in a mapper. - DataSet<String> fin = input.map(new MapFunction<Tuple2<Text,WritableEntity>, String>() { - @Override - public String map(Tuple2<Text, WritableEntity> arg0) throws Exception { - System.err.println("--------------------------------\nKey = "+arg0.f0); - WritableEntity we = arg0.f1; - - for(Map.Entry<String, EntityProperty> prop : we.getProperties().entrySet()) { - System.err.println("key="+prop.getKey() + " ; value (asString)="+prop.getValue().getValueAsString()); - } - - return arg0.f0.toString(); - } - }); - - // emit result (this works only locally) - fin.print(); - - // execute program - env.execute("Azure Example"); - } -} -~~~ - -The example shows how to access an Azure table and turn data into Flink's `DataSet` (more specifically, the type of the set is `DataSet<Tuple2<Text, WritableEntity>>`). With the `DataSet`, you can apply all known transformations to the DataSet. - -## Access MongoDB - -This [GitHub repository documents how to use MongoDB with Apache Flink (starting from 0.7-incubating)](https://github.com/okkam-it/flink-mongodb-test). - -
