Repository: apex-site Updated Branches: refs/heads/asf-site 21739f4ba -> b25c090d9
http://git-wip-us.apache.org/repos/asf/apex-site/blob/b25c090d/docs/malhar-3.7/operators/windowedOperator/index.html ---------------------------------------------------------------------- diff --git a/docs/malhar-3.7/operators/windowedOperator/index.html b/docs/malhar-3.7/operators/windowedOperator/index.html new file mode 100644 index 0000000..6f70d5b --- /dev/null +++ b/docs/malhar-3.7/operators/windowedOperator/index.html @@ -0,0 +1,547 @@ +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + + + <title>Windowed Operator - Apache Apex Malhar Documentation</title> + + + <link rel="shortcut icon" href="../../favicon.ico"> + + + + <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'> + + <link rel="stylesheet" href="../../css/theme.css" type="text/css" /> + <link rel="stylesheet" href="../../css/theme_extra.css" type="text/css" /> + <link rel="stylesheet" href="../../css/highlight.css"> + + + <script> + // Current page data + var mkdocs_page_name = "Windowed Operator"; + var mkdocs_page_input_path = "operators/windowedOperator.md"; + var mkdocs_page_url = "/operators/windowedOperator/"; + </script> + + <script src="../../js/jquery-2.1.1.min.js"></script> + <script src="../../js/modernizr-2.8.3.min.js"></script> + <script type="text/javascript" src="../../js/highlight.pack.js"></script> + <script src="../../js/theme.js"></script> + + +</head> + +<body class="wy-body-for-nav" role="document"> + + <div class="wy-grid-for-nav"> + + + <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav"> + <div class="wy-side-nav-search"> + <a href="../.." class="icon icon-home"> Apache Apex Malhar Documentation</a> + <div role="search"> + <form id ="rtd-search-form" class="wy-form" action="../../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + </form> +</div> + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + <ul class="current"> + + <li> + <li class="toctree-l1 "> + <a class="" href="../..">Apache Apex Malhar</a> + + </li> +<li> + + <li> + <ul class="subnav"> + <li><span>APIs</span></li> + + + + <li class="toctree-l1 "> + <a class="" href="../../apis/calcite/">SQL</a> + + </li> + + + </ul> +<li> + + <li> + <ul class="subnav"> + <li><span>Operators</span></li> + + + + <li class="toctree-l1 "> + <a class="" href="../block_reader/">Block Reader</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../csvformatter/">CSV Formatter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../csvParserOperator/">CSV Parser</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../deduper/">Deduper</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../enricher/">Enricher</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../fsInputOperator/">File Input</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../file_output/">File Output</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../file_splitter/">File Splitter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../filter/">Filter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../AbstractJdbcTransactionableOutputOperator/">Jdbc Output Operator</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../jdbcPollInputOperator/">JDBC Poller Input</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../jmsInputOperator/">JMS Input</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../jsonFormatter/">JSON Formatter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../jsonParser/">JSON Parser</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../kafkaInputOperator/">Kafka Input</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../transform/">Transformer</a> + + </li> + + + + <li class="toctree-l1 current"> + <a class="current" href="./">Windowed Operator</a> + + <ul> + + <li class="toctree-l3"><a href="#windowed-operator">WINDOWED OPERATOR</a></li> + + <li><a class="toctree-l4" href="#introduction">Introduction</a></li> + + <li><a class="toctree-l4" href="#operator-overview">Operator Overview</a></li> + + <li><a class="toctree-l4" href="#keyed-or-not-keyed">Keyed or Not Keyed</a></li> + + <li><a class="toctree-l4" href="#window-option">Window Option</a></li> + + <li><a class="toctree-l4" href="#timestamp-extractor">Timestamp Extractor</a></li> + + <li><a class="toctree-l4" href="#watermarks">Watermarks</a></li> + + <li><a class="toctree-l4" href="#allowed-lateness">Allowed Lateness</a></li> + + <li><a class="toctree-l4" href="#accumulation">Accumulation</a></li> + + <li><a class="toctree-l4" href="#triggers">Triggers</a></li> + + <li><a class="toctree-l4" href="#accumulation-mode">Accumulation Mode</a></li> + + <li><a class="toctree-l4" href="#window-propagation">Window Propagation</a></li> + + <li><a class="toctree-l4" href="#state-storage">State Storage</a></li> + + <li><a class="toctree-l4" href="#merging-two-streams">Merging two streams</a></li> + + <li><a class="toctree-l4" href="#usage-examples">Usage Examples</a></li> + + <li><a class="toctree-l4" href="#advanced-topics">Advanced Topics</a></li> + + + </ul> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../xmlParserOperator/">XML Parser</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../jsonParser/">Json Parser</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../jsonFormatter/">Json Formatter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../transform/">Transform Operator</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../csvformatter/">Csv Formatter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../s3outputmodule/">S3 Output Module</a> + + </li> + + + </ul> +<li> + + </ul> + </div> + + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../..">Apache Apex Malhar Documentation</a> + </nav> + + + <div class="wy-nav-content"> + <div class="rst-content"> + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> + <li><a href="../..">Docs</a> »</li> + + + + <li>Operators »</li> + + + + <li>Windowed Operator</li> + <li class="wy-breadcrumbs-aside"> + + </li> + </ul> + <hr/> +</div> + <div role="main"> + <div class="section"> + + <h1 id="windowed-operator">WINDOWED OPERATOR</h1> +<h2 id="introduction">Introduction</h2> +<p>The <code>WindowedOperator</code> is an operator in the Apex Malhar Library that supports the windowing semantics outlined by Apache Beam, including the notions of watermarks, triggers, accumulation modes, and allowed lateness. It currently supports event time windows, sliding event time windows, session windows, and global window. The reader of this document is encouraged to read this <a href="https://www.oreilly.com/ideas/the-world-beyond-batch-streaming-101";>blog</a> for the basic concepts of streaming applications, and this <a href="https://www.oreilly.com/ideas/the-world-beyond-batch-streaming-102";>blog</a> for Apache Beam's windowing semantics before using this operator.</p> +<p>Our High-Level API supports event-time processing through the WindowedOperator. If you'd like to process tuples based on event time, you are encouraged to use this operator either directly with our DAG-level API, or indirectly through our High-Level API.</p> +<p>It is important to note that the word "windows" in this document is unrelated to "streaming windows" or "application windows" in Apex, which are based on ingression time. For more information about "streaming windows" and "application windows" in Apex, please refer to <a href="http://apex.apache.org/docs/apex/application_development/#streaming-window";>this document</a>.</p> +<h2 id="operator-overview">Operator Overview</h2> +<p>In this document, we will explore the following features in the WindowedOperator.</p> +<ol> +<li>Keyed or Not Keyed</li> +<li>Window Option</li> +<li>Timestamp Extractor</li> +<li>Watermarks</li> +<li>Allowed Lateness</li> +<li>Accumulation</li> +<li>Triggers</li> +<li>Accumulation Mode</li> +<li>Window Propagation</li> +<li>Merging two streams</li> +</ol> +<h2 id="keyed-or-not-keyed">Keyed or Not Keyed</h2> +<p>One of the first things the user of the operator has to decide is whether the operator is keyed (<a href="https://github.com/apache/apex-malhar/blob/master/library/src/main/java/org/apache/apex/malhar/lib/window/impl/KeyedWindowedOperatorImpl.java";>KeyedWindowedOperatorImpl</a>) or not keyed (<a href="https://github.com/apache/apex-malhar/blob/master/library/src/main/java/org/apache/apex/malhar/lib/window/impl/WindowedOperatorImpl.java";>WindowedOperatorImpl</a>). State storage, accumulation and triggers behave differently based on whether or not the operator is keyed.</p> +<p>Here are examples of creating a windowed operator.</p> +<p>Non-keyed:</p> +<pre><code class="java">// Creating a non-keyed windowed operator +WindowedOperatorImpl<InputType, AccumulationType, OutputType> windowedOperator = new WindowedOperatorImpl<>(); +</code></pre> + +<p>Keyed:</p> +<pre><code class="java">// Creating a keyed windowed operator +KeyedWindowedOperatorImpl<KeyType, InputValueType, AccumulationType, OutputValueType> windowedOperator = new KeyedWindowedOperatorImpl<>(); +</code></pre> + +<p>We will cover the concepts of <a href="#state_storage">state storage</a>, <a href="#accumulation">accumulation</a> and <a href="#triggers">triggers</a> later in this document.</p> +<h2 id="window-option">Window Option</h2> +<p>Each incoming tuple of the WindowedOperator is assigned to one or more windows. The <code>WindowOption</code> provides the way to specify what constitutes a window. The following <code>WindowOption</code>s are supported.</p> +<h3 id="globalwindow"><code>GlobalWindow</code></h3> +<p>There is only one window for the entire duration of the application. All tuples are assigned to this one window.</p> +<pre><code class="java">// Setting the global window option +windowedOperator.setWindowOption(new WindowOption.GlobalWindow()); +</code></pre> + +<h3 id="timewindows"><code>TimeWindows</code></h3> +<p>A tuple is assigned to exactly one window based on event time, and each window has a fixed duration. One window is followed immediately by another window, and they do not overlap. As a result, one timestamp can only belong to one window.</p> +<p><img alt="" src="../images/windowedOperator/time-windows.png" /> </p> +<pre><code class="java">// Setting a time window option with a duration of 10 minutes +windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.standardMinutes(10))); +</code></pre> + +<h3 id="slidingtimewindows"><code>SlidingTimeWindows</code></h3> +<p>Similar to <code>TimeWindow</code>, each window has a fixed duration. But it takes an additional duration parameter <code>slideBy</code> which must be smaller than the window duration and the window duration must be divisible by the <code>slideBy</code> duration. Each window overlaps with multiple windows. In this case, since one timestamp belongs to multiple windows, a tuple is assigned to multiple windows. The number of windows a tuple belongs to is exactly the window duration divided by the <code>slideBy</code> duration.</p> +<p><img alt="" src="../images/windowedOperator/sliding-time-windows.png" /> </p> +<pre><code class="java">// Setting a sliding time window option with a duration of 10 minutes and a slideBy duration of 2 minutes +windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.standardMinutes(10)).slideBy(Duration.standardMinutes(2))); +</code></pre> + +<h3 id="sessionwindows"><code>SessionWindows</code></h3> +<p><code>SessionWindow</code>s have variable durations and are based on the key of the tuple. Each tuple is assigned to exactly one window. It takes a duration parameter <code>minGap</code>, which specifies the minimum time gap between two tuples that belong to two <em>different</em> <code>SessionWindows</code> of the same key. <code>minGap</code> is also the duration of the "proto-session" window for a single timestamp, and it is the minimum duration of any session window.</p> +<pre><code class="java">// Setting a session window option with a minimum gap of one hour +windowedOperator.setWindowOption(new WindowOption.SessionWindows(Duration.standardHours(1))); +</code></pre> + +<p>Upon arrival of a tuple, a proto-session window is created. A proto-session window for a tuple is a temporary session window with begin timestamp being the tuple timestamp and the duration being <code>minGap</code>.</p> +<p><img alt="" src="../images/windowedOperator/session-windows-4.png" /></p> +<p>To ensure that there are no two tuples of the same key in different session windows that are less than <code>minGap</code> apart, the <code>WindowedOperator</code> does the following checks:</p> +<h4 id="the-proto-session-window-can-be-fit-into-an-existing-sessionwindow-of-the-same-key-without-change">The proto-session window can be fit into an existing <code>SessionWindow</code> of the same key without change</h4> +<p>The new tuple is simply applied to the state of the existing <code>SessionWindow</code>.</p> +<p><img alt="" src="../images/windowedOperator/session-windows-3.png" /></p> +<h4 id="the-proto-session-window-overlaps-with-two-existing-session-windows-of-the-same-key">The proto-session window overlaps with two existing session windows of the same key</h4> +<p>A new Session Window is created with the merged state of the two existing <code>SessionWindow</code>s, plus the new tuple. The two existing <code>SessionWindow</code>s will be deleted and retraction triggers for the two deleted windows will be fired. (Please see <a href="#triggers">here</a> for details on <code>Trigger</code>s)</p> +<p><img alt="" src="../images/windowedOperator/session-windows-1.png" /> </p> +<h4 id="the-proto-session-window-overlaps-with-one-existing-session-window-of-the-same-key">The proto-session window overlaps with one existing session window of the same key</h4> +<p>A new <code>SessionWindow</code> is created with the state of the existing <code>SessionWindow</code>, plus the new tuple, with a longer duration than the existing <code>SessionWindow</code> and possibly an earlier begin timestamp to cover the new tuple. The existing <code>SessionWindow</code> will be deleted and a retraction trigger for the old window will be fired.</p> +<p><img alt="" src="../images/windowedOperator/session-windows-2.png" /> </p> +<h4 id="all-of-the-above-checks-return-false">All of the above checks return false</h4> +<p>The proto-session window is in effect and the new tuple is assigned to that window.</p> +<h2 id="timestamp-extractor">Timestamp Extractor</h2> +<p>The <code>WindowedOperator</code> expects a timestamp extractor. This is for <code>WindowedOperator</code> to extract the timestamp from the tuple for window assignment.</p> +<pre><code class="java">// Setting a time extractor +windowedOperator.setTimestampExtractor(new Function<InputTupleType, Long>() +{ + @Override + public Long apply(InputTupleType tuple) + { + return tuple.timestamp; + } +}); + +</code></pre> + +<h2 id="watermarks">Watermarks</h2> +<p>Watermarks are control tuples that include a timestamp. A watermark tells <code>WindowedOperator</code> that all windows that lie completely before the given timestamp are considered late, and the rest of the windows are considered early. </p> +<h3 id="fixed-watermark">Fixed Watermark</h3> +<p>If watermarks are not available from upstream, the user of the WindowedOperator can set a fixed watermark. The fixed watermark represents the number of milliseconds before the timestamp derived from the Apex streaming window ID. Note that the Apex streaming window ID is an implicit timestamp that more or less represents the ingression time of the tuple.</p> +<pre><code class="java">// Setting a fixed watermark that is 10 seconds behind the ingression time +windowedOperator.setFixedWatermark(10000); +</code></pre> + +<h2 id="allowed-lateness">Allowed Lateness</h2> +<p>Allowed Lateness specifies the lateness horizon from the watermark. If a tuple has a timestamp that lies beyond the lateness horizon, it is dropped by the <code>WindowedOperator</code>. Also, if a window completely lies beyond the lateness horizon as a result of the arrival of a new watermark, the window along with its state is purged from <code>WindowedOperator</code>.</p> +<p><img alt="" src="../images/windowedOperator/allowed-lateness.png" /> </p> +<pre><code class="java">// Setting allowed lateness to be one hour +windowedOperator.setAllowedLateness(Duration.standardHours(1)); +</code></pre> + +<h2 id="accumulation">Accumulation</h2> +<p>The Accumulation object tells the <code>WindowedOperator</code> how the operator state is accumulated. It tells the <code>WindowedOperator</code> what to do with its state upon arrival of an incoming tuple. This is where the business logic goes. Please refer to the interface definition <a href="https://github.com/apache/apex-malhar/blob/master/library/src/main/java/org/apache/apex/malhar/lib/window/Accumulation.java";>here</a> in github. For non-keyed WindowedOperator, the state is per window. For keyed WindowedOperator, the state is per key per window.</p> +<pre><code class="java">// Setting the accumulation to be the sum for longs, assuming both the input type and the output type are a long +windowedOperator.setAccumulation(new SumLong()); +</code></pre> + +<p>The user of this operator can use one of the existing accumulation implementations <a href="https://github.com/apache/apex-malhar/tree/master/library/src/main/java/org/apache/apex/malhar/lib/window/accumulation";>here</a>, or provides their own custom accumulation that reflects their business logic. </p> +<h2 id="triggers">Triggers</h2> +<p>Triggers are tuples emitted to downstream by the <code>WindowedOperator</code>. The data in the tuples are based on the state of <code>WindowedOperator</code> governed by the Accumulation object. There are two types of trigger: time-based triggers and count-based triggers. Time-based triggers are triggers that get fired in a regular time interval, and count-based triggers are triggers that get fired based on the number of tuples received. The user of WindowedOperator can specify different triggers for windows that are early or late based on the watermark.</p> +<p>Also, by default, a trigger is fired for a window when the window is flipped from being early to being late. This is also called an "on-time" trigger.</p> +<pre><code class="java">// Creating a trigger option that tells the windowed operator to fire a trigger when the window is flipped from early to late, and fire a trigger every 10 seconds when it's early, and fire a trigger every time a tuple is received when it's late +TriggerOption triggerOption = TriggerOption.AtWatermark().withEarlyFiringsAtEvery(Duration.standardSeconds(10)).withLateFiringsAtEvery(1); + +// Setting the trigger option for the windowed operator +windowedOperator.setTriggerOption(triggerOption); +</code></pre> + +<p>Note that for the non-keyed <code>WindowedOperator</code>, triggers are fired on a per-window basis. For the keyed <code>WindowedOperator</code>, triggers are fired on a per-key-per-window basis.</p> +<p>There is also an option the user can set (<code>fireOnlyUpdatedPanes</code>) to make the <code>WindowedOperator</code> not fire a trigger if the trigger value is the same as the value of the previous trigger. </p> +<pre><code class="java">// Telling the windowed operator to fire a trigger only if the state has been changed since the last trigger +triggerOption.fireOnlyUpdatedPanes(); +</code></pre> + +<h2 id="accumulation-mode">Accumulation Mode</h2> +<p>Accumulation Mode tells the operator what to do with the state of the window when a trigger is fired. +There are three supported accumulation mode: <code>ACCUMULATING</code>, <code>DISCARDING</code>, and <code>ACCUMULATING_AND_DISCARDING</code>.</p> +<ul> +<li><code>ACCUMULATING</code>: The state of the window is preserved until purged</li> +<li><code>DISCARDING</code>: The state of the window is discarded after firing of a trigger</li> +<li><code>ACCUMULATING_AND_RETRACTING</code>: The state of the window is preserved until purged, but if the state has changed upon a trigger compared to the previous trigger, an additional retraction trigger is fired.</li> +</ul> +<pre><code class="java">// Setting accumulation mode to be ACCUMULATING +triggerOption.accumulatingFiredPanes(); + +// Setting accumulation mode to be DISCARDING +triggerOption.discardingFiredPanes(); + +// Setting accumulation mode to be ACCUMULATING_AND_RETRACTING +triggerOption.accumulatingAndRetractingFiredPanes(); +</code></pre> + +<h2 id="window-propagation">Window Propagation</h2> +<p>It is possible to chain multiple instances of <code>WindowedOperator</code> and have only the most upstream instance assign the windows and have all downstream instances inherit the same windows of the triggers from the upstream instance. If WindowOption is <code>null</code> (i.e. <code>setWindowOption</code> is not called), the <code>WindowedOperator</code> assumes that the incoming tuples are <code>WindowedTuple</code>s that contain the information of the window assignment for each tuple.</p> +<h2 id="state-storage">State Storage</h2> +<p>One of the most important elements of the <code>WindowedOperator</code> is the state storage. Each window in the operator (or each window/key pair if the operator is keyed) has its own state and how the state is stored and checkpointed is likely to be the most important factor for performance.</p> +<p>The <code>WindowedOperator</code> currently supports two different state storage mechanisms.</p> +<p><a href="https://github.com/apache/apex-malhar/blob/master/library/src/main/java/org/apache/apex/malhar/lib/window/impl/InMemoryWindowedStorage.java";>In-Memory Windowed Storage</a> stores the operator state only in memory and the entire state is copied to DFS at checkpoint. This storage is useful only if the state is expected to be small and the cardinality of valid windows and keys is small.</p> +<pre><code class="java">// Setting the data storage for windowed operator to be an in-memory storage +windowedOperator.setDataStorage(new InMemoryWindowedStorage<AccumulationType>()); +</code></pre> + +<p><a href="https://github.com/apache/apex-malhar/blob/master/library/src/main/java/org/apache/apex/malhar/lib/window/impl/SpillableWindowedPlainStorage.java";>Spillable Windowed Storage</a> stores the operator state in DFS with a cache in memory. This storage mechanism handles large states and incremental checkpointing. </p> +<pre><code class="java">// Setting the data storage for windowed operator to be a spillable storage +SpillableWindowedPlainStorage<MutableLong> dataStorage = new SpillableWindowedPlainStorage<>(); +// setup of the spillable storage omitted here for brevity +windowedOperator.setDataStorage(dataStorage); +</code></pre> + +<h2 id="merging-two-streams">Merging two streams</h2> +<p>The <code>WindowedMergeOperator</code> is a <code>WindowedOperator</code> that takes two incoming data streams. It takes a <a href="https://github.com/apache/apex-malhar/blob/master/library/src/main/java/org/apache/apex/malhar/lib/window/MergeAccumulation.java";><code>MergeAccumulation</code></a> instead of a regular Accumulation. The user of this operator can implement their custom merge or join accumulation based on their business logic. Examples of this type of accumulation are <a href="https://github.com/apache/apex-malhar/blob/master/library/src/main/java/org/apache/apex/malhar/lib/window/accumulation/InnerJoin.java";>InnerJoin</a> and <a href="https://github.com/apache/apex-malhar/blob/master/library/src/main/java/org/apache/apex/malhar/lib/window/accumulation/CoGroup.java";>Cogroup</a>.</p> +<p>The <code>WindowedMergeOperator</code> has its own watermark. Its watermark timestamp is the earlier watermark timestamp between the two input streams. When that value changes upon the arrival of incoming watermarks, a watermark control tuple with that timestamp value will be fired to downstream.</p> +<h2 id="usage-examples">Usage Examples</h2> +<p>For an example usage of the <code>WindowedOperator</code> via the High level API, click <a href="https://github.com/apache/apex-malhar/blob/master/examples/highlevelapi/src/main/java/org/apache/apex/malhar/stream/sample/WindowedWordCount.java";>here</a>.</p> +<p>For an example usage of the <code>WindowedOperator</code> via the DAG level API, click <a href="https://github.com/apache/apex-malhar/blob/master/library/src/test/java/org/apache/apex/malhar/lib/window/sample/wordcount/Application.java";>here</a>.</p> +<h2 id="advanced-topics">Advanced Topics</h2> +<h3 id="idempotency-considerations">Idempotency Considerations</h3> +<p>For the <code>WindowedOperator</code> to be <a href="http://apex.apache.org/docs/apex/development_best_practices/#idempotence";>idempotent</a>, both data tuples and watermarks must be emitted deterministically. i.e. When replaying from a checkpoint, the same tuples and watermarks must be emitted in the same Apex streaming window as before the failure happens.</p> +<p>In order to achieve this, the <code>WindowedOperator</code> has the following behavior:</p> +<ol> +<li> +<p>The time-based triggers are fired based on the implicit timestamp from the Apex streaming windows, not based on the wall clock time. And the time-based triggers are fired only at the Apex streaming window boundary. This means that the Apex streaming window width should be smaller than or equal to the interval duration of any time-based trigger, and ideally the interval duration of the time-based trigger can be expressed as an integer multiple of the Apex streaming window width.</p> +</li> +<li> +<p>The processing of the incoming watermarks is only done at the Apex streaming window boundary. This includes the change of status from early to late for windows that lie beyond the watermark timestamp, the purging of windows from the state for windows that lie completely beyond the lateness horizon, and the propagation of watermark tuples to downstream.</p> +</li> +</ol> + + </div> + </div> + <footer> + + <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> + + <a href="../xmlParserOperator/" class="btn btn-neutral float-right" title="XML Parser">Next <span class="icon icon-circle-arrow-right"></span></a> + + + <a href="../transform/" class="btn btn-neutral" title="Transformer"><span class="icon icon-circle-arrow-left"></span> Previous</a> + + </div> + + + <hr/> + + <div role="contentinfo"> + <!-- Copyright etc --> + + </div> + + Built with <a href="http://www.mkdocs.org";>MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme";>theme</a> provided by <a href="https://readthedocs.org";>Read the Docs</a>. +</footer> + + </div> + </div> + + </section> + + </div> + +<div class="rst-versions" role="note" style="cursor: pointer"> + <span class="rst-current-version" data-toggle="rst-current-version"> + + + <span><a href="../transform/" style="color: #fcfcfc;">« Previous</a></span> + + + <span style="margin-left: 15px"><a href="../xmlParserOperator/" style="color: #fcfcfc">Next »</a></span> + + </span> +</div> + +</body> +</html> http://git-wip-us.apache.org/repos/asf/apex-site/blob/b25c090d/docs/malhar-3.7/operators/xmlParserOperator/index.html ---------------------------------------------------------------------- diff --git a/docs/malhar-3.7/operators/xmlParserOperator/index.html b/docs/malhar-3.7/operators/xmlParserOperator/index.html new file mode 100644 index 0000000..6e69836 --- /dev/null +++ b/docs/malhar-3.7/operators/xmlParserOperator/index.html @@ -0,0 +1,482 @@ +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + + + <title>XML Parser - Apache Apex Malhar Documentation</title> + + + <link rel="shortcut icon" href="../../favicon.ico"> + + + + <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'> + + <link rel="stylesheet" href="../../css/theme.css" type="text/css" /> + <link rel="stylesheet" href="../../css/theme_extra.css" type="text/css" /> + <link rel="stylesheet" href="../../css/highlight.css"> + + + <script> + // Current page data + var mkdocs_page_name = "XML Parser"; + var mkdocs_page_input_path = "operators/xmlParserOperator.md"; + var mkdocs_page_url = "/operators/xmlParserOperator/"; + </script> + + <script src="../../js/jquery-2.1.1.min.js"></script> + <script src="../../js/modernizr-2.8.3.min.js"></script> + <script type="text/javascript" src="../../js/highlight.pack.js"></script> + <script src="../../js/theme.js"></script> + + +</head> + +<body class="wy-body-for-nav" role="document"> + + <div class="wy-grid-for-nav"> + + + <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav"> + <div class="wy-side-nav-search"> + <a href="../.." class="icon icon-home"> Apache Apex Malhar Documentation</a> + <div role="search"> + <form id ="rtd-search-form" class="wy-form" action="../../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + </form> +</div> + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + <ul class="current"> + + <li> + <li class="toctree-l1 "> + <a class="" href="../..">Apache Apex Malhar</a> + + </li> +<li> + + <li> + <ul class="subnav"> + <li><span>APIs</span></li> + + + + <li class="toctree-l1 "> + <a class="" href="../../apis/calcite/">SQL</a> + + </li> + + + </ul> +<li> + + <li> + <ul class="subnav"> + <li><span>Operators</span></li> + + + + <li class="toctree-l1 "> + <a class="" href="../block_reader/">Block Reader</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../csvformatter/">CSV Formatter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../csvParserOperator/">CSV Parser</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../deduper/">Deduper</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../enricher/">Enricher</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../fsInputOperator/">File Input</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../file_output/">File Output</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../file_splitter/">File Splitter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../filter/">Filter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../AbstractJdbcTransactionableOutputOperator/">Jdbc Output Operator</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../jdbcPollInputOperator/">JDBC Poller Input</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../jmsInputOperator/">JMS Input</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../jsonFormatter/">JSON Formatter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../jsonParser/">JSON Parser</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../kafkaInputOperator/">Kafka Input</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../transform/">Transformer</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../windowedOperator/">Windowed Operator</a> + + </li> + + + + <li class="toctree-l1 current"> + <a class="current" href="./">XML Parser</a> + + <ul> + + <li class="toctree-l3"><a href="#xml-parser">Xml Parser</a></li> + + <li><a class="toctree-l4" href="#operator-objective">Operator Objective</a></li> + + <li><a class="toctree-l4" href="#class-diagram">Class Diagram</a></li> + + <li><a class="toctree-l4" href="#operator-information">Operator Information</a></li> + + <li><a class="toctree-l4" href="#properties-attributes-and-ports">Properties, Attributes and Ports</a></li> + + <li><a class="toctree-l4" href="#partitioning">Partitioning</a></li> + + <li><a class="toctree-l4" href="#example">Example</a></li> + + + </ul> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../jsonParser/">Json Parser</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../jsonFormatter/">Json Formatter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../transform/">Transform Operator</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../csvformatter/">Csv Formatter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="../s3outputmodule/">S3 Output Module</a> + + </li> + + + </ul> +<li> + + </ul> + </div> + + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../..">Apache Apex Malhar Documentation</a> + </nav> + + + <div class="wy-nav-content"> + <div class="rst-content"> + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> + <li><a href="../..">Docs</a> »</li> + + + + <li>Operators »</li> + + + + <li>XML Parser</li> + <li class="wy-breadcrumbs-aside"> + + </li> + </ul> + <hr/> +</div> + <div role="main"> + <div class="section"> + + <h1 id="xml-parser">Xml Parser</h1> +<h2 id="operator-objective">Operator Objective</h2> +<p>The XmlParser operator parses XML records and constructs POJOs ("Plain Old Java Objects") from them. The operator also emits each record as a DOM Document if the relevant output port is connected. User can also provide a XSD (XML Schema Definition) to validate incoming XML records. Valid records will be emitted as POJOs / DOM Document while invalid ones are emitted on error port with an error message if the error port is connected.</p> +<p>XmlParser is <strong>idempotent</strong>, <strong>fault-tolerant</strong> and <strong>statically/dynamically partitionable</strong>.</p> +<h2 id="class-diagram">Class Diagram</h2> +<p><img alt="" src="../images/xmlParser/XmlParser.png" /></p> +<h2 id="operator-information">Operator Information</h2> +<ol> +<li>Operator location: <strong><em>malhar-library</em></strong></li> +<li>Available since: <strong><em>3.2.0</em></strong></li> +<li>Operator state: <strong><em>Evolving</em></strong></li> +<li>Java Package: <a href="https://github.com/apache/apex-malhar/blob/master/library/src/main/java/com/datatorrent/lib/parser/XmlParser.java";>com.datatorrent.lib.parser.XmlParser</a></li> +</ol> +<h2 id="properties-attributes-and-ports">Properties, Attributes and Ports</h2> +<h3 id="properties-of-xml-parser"><a name="props"></a>Properties of Xml Parser</h3> +<table> +<thead> +<tr> +<th><strong>Property</strong></th> +<th><strong>Description</strong></th> +<th><strong>Type</strong></th> +<th><strong>Mandatory</strong></th> +<th><strong>Default Value</strong></th> +</tr> +</thead> +<tbody> +<tr> +<td><em>schemaXSDFile</em></td> +<td>[XSD] describing XML data. Incoming records can be validated using the schemaXSDFile. If the data is not as per the requirements specified in schemaXSDFile, they are emitted on the error port. This is an optional property. If the XSD is not provided, incoming tuples are simply converted to POJOs or DOM Documents without any validations</td> +<td>String</td> +<td>No</td> +<td>N/A</td> +</tr> +</tbody> +</table> +<h3 id="platform-attributes-that-influence-operator-behavior">Platform Attributes that influence operator behavior</h3> +<table> +<thead> +<tr> +<th><strong>Attribute</strong></th> +<th><strong>Description</strong></th> +<th><strong>Type</strong></th> +<th><strong>Mandatory</strong></th> +</tr> +</thead> +<tbody> +<tr> +<td><em>out.TUPLE_CLASS</em></td> +<td>TUPLE_CLASS attribute on output port which tells operator the class of POJO which needs to be emitted. The name of the field members of the class must match with the names in incoming POJO. The operator ignores unknown properties i.e. fields present in POJO but not in TUPLE_CLASS or vice versa.</td> +<td>Class or FQCN</td> +<td>Yes</td> +</tr> +</tbody> +</table> +<h3 id="ports">Ports</h3> +<table> +<thead> +<tr> +<th><strong>Port</strong></th> +<th><strong>Description</strong></th> +<th><strong>Type</strong></th> +<th><strong>Mandatory</strong></th> +</tr> +</thead> +<tbody> +<tr> +<td><em>in</em></td> +<td>Tuples that needs to be parsed are received on this port</td> +<td>byte[]</td> +<td>Yes</td> +</tr> +<tr> +<td><em>out</em></td> +<td>Valid Tuples that are emitted as pojo. Tuples are converted to POJO only if the port is connected.</td> +<td>Object (POJO)</td> +<td>No</td> +</tr> +<tr> +<td><em>parsedOutput</em></td> +<td>Valid Tuples that are emitted as DOM Document. Tuples are converted to DOM Document only if the port is connected.</td> +<td>DOM Document</td> +<td>No</td> +</tr> +<tr> +<td><em>err</em></td> +<td>Invalid Tuples are emitted with error message. Invalid tuples are discarded if the port is not connected.</td> +<td>KeyValPair <String, String></td> +<td>No</td> +</tr> +</tbody> +</table> +<h2 id="partitioning">Partitioning</h2> +<p>XML Parser is both statically and dynamically partitionable.</p> +<h3 id="static-partitioning">Static Partitioning</h3> +<p>This can be achieved in 2 ways</p> +<ol> +<li>Specifying the partitioner and number of partitions in the 'populateDAG()' method.</li> +</ol> +<pre><code class="java">XmlParser xmlParser = dag.addOperator("xmlParser", XmlParser.class); +StatelessPartitioner<XmlParser> partitioner1 = new StatelessPartitioner<XmlParser>(2); +dag.setAttribute(xmlParser, Context.OperatorContext.PARTITIONER, partitioner1 ); +</code></pre> + +<ol> +<li>Specifying the partitioner and number of partitions in properties file.</li> +</ol> +<pre><code class="xml"> <property> + <name>dt.operator.{OperatorName}.attr.PARTITIONER</name> + <value>com.datatorrent.common.partitioner.StatelessPartitioner:2</value> + </property> +</code></pre> + +<p>where {OperatorName} is the name of the XmlParser operator. + Above lines will partition XmlParser statically 2 times. Above value can be changed accordingly to change the number of static partitions.</p> +<h3 id="dynamic-partitioning">Dynamic Partitioning</h3> +<p>XmlParser can be dynamically partitioned using an out-of-the-box partitioner:</p> +<h4 id="throughput-based">Throughput based</h4> +<p>Following code can be added to 'populateDAG' method of application to dynamically partition XmlParser:</p> +<pre><code class="java">XmlParser xmlParser = dag.addOperator("xmlParser", XmlParser.class); +StatelessThroughputBasedPartitioner<XmlParser> partitioner = new StatelessThroughputBasedPartitioner<>(); +partitioner.setCooldownMillis(conf.getLong("dt.cooldown", 10000)); +partitioner.setMaximumEvents(conf.getLong("dt.maxThroughput", 30000)); +partitioner.setMinimumEvents(conf.getLong("dt.minThroughput", 10000)); +dag.setAttribute(xmlParser, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[]{partitioner})); +dag.setAttribute(xmlParser, OperatorContext.PARTITIONER, partitioner); +</code></pre> + +<p>Above code will dynamically partition XmlParser when the throughput changes. +If the overall throughput of XmlParser goes beyond 30000 or less than 10000, the platform will repartition XmlParser +to balance throughput of a single partition to be between 10000 and 30000. +'dt.cooldown' of 10000 will be used as the threshold time for which the throughput change is observed.</p> +<h2 id="example">Example</h2> +<p>Example for Xml Parser can be found at: <a href="https://github.com/DataTorrent/examples/tree/master/tutorials/parser";>https://github.com/DataTorrent/examples/tree/master/tutorials/parser</a></p> + + </div> + </div> + <footer> + + <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> + + <a href="../jsonParser/" class="btn btn-neutral float-right" title="Json Parser">Next <span class="icon icon-circle-arrow-right"></span></a> + + + <a href="../windowedOperator/" class="btn btn-neutral" title="Windowed Operator"><span class="icon icon-circle-arrow-left"></span> Previous</a> + + </div> + + + <hr/> + + <div role="contentinfo"> + <!-- Copyright etc --> + + </div> + + Built with <a href="http://www.mkdocs.org";>MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme";>theme</a> provided by <a href="https://readthedocs.org";>Read the Docs</a>. +</footer> + + </div> + </div> + + </section> + + </div> + +<div class="rst-versions" role="note" style="cursor: pointer"> + <span class="rst-current-version" data-toggle="rst-current-version"> + + + <span><a href="../windowedOperator/" style="color: #fcfcfc;">« Previous</a></span> + + + <span style="margin-left: 15px"><a href="../jsonParser/" style="color: #fcfcfc">Next »</a></span> + + </span> +</div> + +</body> +</html> http://git-wip-us.apache.org/repos/asf/apex-site/blob/b25c090d/docs/malhar-3.7/search.html ---------------------------------------------------------------------- diff --git a/docs/malhar-3.7/search.html b/docs/malhar-3.7/search.html new file mode 100644 index 0000000..fad4333 --- /dev/null +++ b/docs/malhar-3.7/search.html @@ -0,0 +1,319 @@ +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + + + <title>Apache Apex Malhar Documentation</title> + + + <link rel="shortcut icon" href="favicon.ico"> + + + + <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'> + + <link rel="stylesheet" href="./css/theme.css" type="text/css" /> + <link rel="stylesheet" href="./css/theme_extra.css" type="text/css" /> + <link rel="stylesheet" href="./css/highlight.css"> + + + <script src="./js/jquery-2.1.1.min.js"></script> + <script src="./js/modernizr-2.8.3.min.js"></script> + <script type="text/javascript" src="./js/highlight.pack.js"></script> + <script src="./js/theme.js"></script> + <script>var base_url = '.';</script> + <script data-main="./mkdocs/js/search.js" src="./mkdocs/js/require.js"></script> + + + +</head> + +<body class="wy-body-for-nav" role="document"> + + <div class="wy-grid-for-nav"> + + + <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav"> + <div class="wy-side-nav-search"> + <a href="." class="icon icon-home"> Apache Apex Malhar Documentation</a> + <div role="search"> + <form id ="rtd-search-form" class="wy-form" action="./search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + </form> +</div> + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + <ul class="current"> + + <li> + <li class="toctree-l1 "> + <a class="" href=".">Apache Apex Malhar</a> + + </li> +<li> + + <li> + <ul class="subnav"> + <li><span>APIs</span></li> + + + + <li class="toctree-l1 "> + <a class="" href="apis/calcite/">SQL</a> + + </li> + + + </ul> +<li> + + <li> + <ul class="subnav"> + <li><span>Operators</span></li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/block_reader/">Block Reader</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/csvformatter/">CSV Formatter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/csvParserOperator/">CSV Parser</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/deduper/">Deduper</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/enricher/">Enricher</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/fsInputOperator/">File Input</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/file_output/">File Output</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/file_splitter/">File Splitter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/filter/">Filter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/AbstractJdbcTransactionableOutputOperator/">Jdbc Output Operator</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/jdbcPollInputOperator/">JDBC Poller Input</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/jmsInputOperator/">JMS Input</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/jsonFormatter/">JSON Formatter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/jsonParser/">JSON Parser</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/kafkaInputOperator/">Kafka Input</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/transform/">Transformer</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/windowedOperator/">Windowed Operator</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/xmlParserOperator/">XML Parser</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/jsonParser/">Json Parser</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/jsonFormatter/">Json Formatter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/transform/">Transform Operator</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/csvformatter/">Csv Formatter</a> + + </li> + + + + <li class="toctree-l1 "> + <a class="" href="operators/s3outputmodule/">S3 Output Module</a> + + </li> + + + </ul> +<li> + + </ul> + </div> + + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href=".">Apache Apex Malhar Documentation</a> + </nav> + + + <div class="wy-nav-content"> + <div class="rst-content"> + <div role="navigation" aria-label="breadcrumbs navigation"> + <ul class="wy-breadcrumbs"> + <li><a href=".">Docs</a> »</li> + + + <li class="wy-breadcrumbs-aside"> + + </li> + </ul> + <hr/> +</div> + <div role="main"> + <div class="section"> + + + <h1 id="search">Search Results</h1> + + <form id="content_search" action="search.html"> + <span role="status" aria-live="polite" class="ui-helper-hidden-accessible"></span> + <input name="q" id="mkdocs-search-query" type="text" class="search_input search-query ui-autocomplete-input" placeholder="Search the Docs" autocomplete="off" autofocus> + </form> + + <div id="mkdocs-search-results"> + Searching... + </div> + + + </div> + </div> + <footer> + + + <hr/> + + <div role="contentinfo"> + <!-- Copyright etc --> + + </div> + + Built with <a href="http://www.mkdocs.org";>MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme";>theme</a> provided by <a href="https://readthedocs.org";>Read the Docs</a>. +</footer> + + </div> + </div> + + </section> + + </div> + +<div class="rst-versions" role="note" style="cursor: pointer"> + <span class="rst-current-version" data-toggle="rst-current-version"> + + + + </span> +</div> + +</body> +</html> http://git-wip-us.apache.org/repos/asf/apex-site/blob/b25c090d/docs/malhar-3.7/searchbox.html ---------------------------------------------------------------------- diff --git a/docs/malhar-3.7/searchbox.html b/docs/malhar-3.7/searchbox.html new file mode 100644 index 0000000..177fcb3 --- /dev/null +++ b/docs/malhar-3.7/searchbox.html @@ -0,0 +1,5 @@ +<div role="search"> + <form id ="rtd-search-form" class="wy-form" action="{{ base_url }}/search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + </form> +</div> http://git-wip-us.apache.org/repos/asf/apex-site/blob/b25c090d/docs/malhar-3.7/sitemap.xml ---------------------------------------------------------------------- diff --git a/docs/malhar-3.7/sitemap.xml b/docs/malhar-3.7/sitemap.xml new file mode 100644 index 0000000..10c62ab --- /dev/null +++ b/docs/malhar-3.7/sitemap.xml @@ -0,0 +1,164 @@ +<?xml version="1.0" encoding="UTF-8"?> +<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9";> + + + <url> + <loc>/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + + + + <url> + <loc>/apis/calcite/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + + + + + <url> + <loc>/operators/block_reader/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/csvformatter/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/csvParserOperator/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/deduper/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/enricher/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/fsInputOperator/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/file_output/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/file_splitter/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/filter/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/AbstractJdbcTransactionableOutputOperator/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/jdbcPollInputOperator/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/jmsInputOperator/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/jsonFormatter/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/jsonParser/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/kafkaInputOperator/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/transform/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/windowedOperator/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/xmlParserOperator/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/jsonParser/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/jsonFormatter/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/transform/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/csvformatter/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + <url> + <loc>/operators/s3outputmodule/</loc> + <lastmod>2017-03-27</lastmod> + <changefreq>daily</changefreq> + </url> + + + +</urlset> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/apex-site/blob/b25c090d/docs/malhar-3.7/toc.html ---------------------------------------------------------------------- diff --git a/docs/malhar-3.7/toc.html b/docs/malhar-3.7/toc.html new file mode 100644 index 0000000..6cd2fc9 --- /dev/null +++ b/docs/malhar-3.7/toc.html @@ -0,0 +1,23 @@ +{% if nav_item.children %} + <ul class="subnav"> + <li><span>{{ nav_item.title }}</span></li> + + {% for nav_item in nav_item.children %} + {% include 'toc.html' %} + {% endfor %} + </ul> +{% else %} + <li class="toctree-l1 {% if nav_item.active%}current{%endif%}"> + <a class="{% if nav_item.active%}current{%endif%}" href="{{ nav_item.url }}">{{ nav_item.title }}</a> + {% if nav_item == current_page %} + <ul> + {% for toc_item in toc %} + <li class="toctree-l3"><a href="{{ toc_item.url }}">{{ toc_item.title }}</a></li> + {% for toc_item in toc_item.children %} + <li><a class="toctree-l4" href="{{ toc_item.url }}">{{ toc_item.title }}</a></li> + {% endfor %} + {% endfor %} + </ul> + {% endif %} + </li> +{% endif %} http://git-wip-us.apache.org/repos/asf/apex-site/blob/b25c090d/docs/malhar-3.7/versions.html ---------------------------------------------------------------------- diff --git a/docs/malhar-3.7/versions.html b/docs/malhar-3.7/versions.html new file mode 100644 index 0000000..d12d197 --- /dev/null +++ b/docs/malhar-3.7/versions.html @@ -0,0 +1,15 @@ +<div class="rst-versions" role="note" style="cursor: pointer"> + <span class="rst-current-version" data-toggle="rst-current-version"> + {% if repo_name == 'GitHub' %} + <a href="{{ repo_url }}" class="icon icon-github" style="float: left; color: #fcfcfc"> GitHub</a> + {% elif repo_name == 'Bitbucket' %} + <a href="{{ repo_url }}" class="icon icon-bitbucket" style="float: left; color: #fcfcfc"> BitBucket</a> + {% endif %} + {% if previous_page %} + <span><a href="{{ previous_page.url }}" style="color: #fcfcfc;">« Previous</a></span> + {% endif %} + {% if next_page %} + <span style="margin-left: 15px"><a href="{{ next_page.url }}" style="color: #fcfcfc">Next »</a></span> + {% endif %} + </span> +</div>
