http://git-wip-us.apache.org/repos/asf/bahir-website/blob/8f21f868/content/docs/spark/2.3.0/spark-sql-streaming-mqtt/index.html ---------------------------------------------------------------------- diff --git a/content/docs/spark/2.3.0/spark-sql-streaming-mqtt/index.html b/content/docs/spark/2.3.0/spark-sql-streaming-mqtt/index.html new file mode 100644 index 0000000..ac139cb --- /dev/null +++ b/content/docs/spark/2.3.0/spark-sql-streaming-mqtt/index.html @@ -0,0 +1,479 @@ + + +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Spark Structured Streaming MQTT</title> + <meta name="description" content="Spark Structured Streaming MQTT"> + <meta name="author" content=""> + + <!-- Enable responsive viewport --> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <!-- Le HTML5 shim, for IE6-8 support of HTML elements --> + <!--[if lt IE 9]> + <script src="http://html5shim.googlecode.com/svn/trunk/html5.js";></script> + <![endif]--> + + <!-- Le styles --> + <link href="/assets/themes/apache-clean/bootstrap/css/bootstrap.css" rel="stylesheet"> + <link href="/assets/themes/apache-clean/css/style.css?body=1" rel="stylesheet" type="text/css"> + <link href="/assets/themes/apache-clean/css/syntax.css" rel="stylesheet" type="text/css" media="screen" /> + <!-- Le fav and touch icons --> + <!-- Update these with your own images + <link rel="shortcut icon" href="images/favicon.ico"> + <link rel="apple-touch-icon" href="images/apple-touch-icon.png"> + <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png"> + <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png"> + --> + + <!-- make tables sortable by adding class tag "sortable" to table elements --> + <script src="http://www.kryogenix.org/code/browser/sorttable/sorttable.js";></script> + + + </head> + + <body> + + + +<!-- Navigation --> +<div id="nav-bar"> + <nav id="nav-container" class="navbar navbar-inverse " role="navigation"> + <div class="container"> + <!-- Brand and toggle get grouped for better mobile display --> + + <div class="navbar-header page-scroll"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand page-scroll" href="/#home">Home</a> + </div> + <!-- Collect the nav links, forms, and other content for toggling --> + <nav class="navbar-collapse collapse" role="navigation"> + <ul class="nav navbar-nav"> + + + + <li id="download"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Download<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/downloads/spark" target="_self">Bahir Spark Extensions</a></li> + + + <li><a href="/downloads/flink" target="_self">Bahir Flink Extensions</a></li> + + </ul> + + </li> + + + + + <li id="community"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Community<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/community" target="_self">Get Involved</a></li> + + + <li><a href="/contributing" target="_self">Contributing</a></li> + + + <li><a href="/contributing-extensions" target="_self">Contributing Extensions</a></li> + + + <li><a href="https://issues.apache.org/jira/browse/BAHIR"; target="_blank">Issue Tracker</a></li> + + + <li><a href="/community#source-code" target="_self">Source Code</a></li> + + + <li><a href="/community-members" target="_self">Project Committers</a></li> + + </ul> + + </li> + + + + + <li id="documentation"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Documentation<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/docs/spark/overview" target="_self">Bahir Spark Extensions</a></li> + + + <li><a href="/docs/flink/overview" target="_self">Bahir Flink Extensions</a></li> + + </ul> + + </li> + + + + + <li id="github"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">GitHub<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="https://github.com/apache/bahir"; target="_blank">Bahir Spark Extensions</a></li> + + + <li><a href="https://github.com/apache/bahir-flink"; target="_blank">Bahir Flink Extensions</a></li> + + + <li><a href="https://github.com/apache/bahir-website"; target="_blank">Bahir Website</a></li> + + </ul> + + </li> + + + + + <li id="apache"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Apache<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="http://www.apache.org/foundation/how-it-works.html"; target="_blank">Apache Software Foundation</a></li> + + + <li><a href="http://www.apache.org/licenses/"; target="_blank">Apache License</a></li> + + + <li><a href="http://www.apache.org/foundation/sponsorship"; target="_blank">Sponsorship</a></li> + + + <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> + + + <li><a href="/privacy-policy" target="_self">Privacy Policy</a></li> + + </ul> + + </li> + + + </ul> + </nav><!--/.navbar-collapse --> + <!-- /.navbar-collapse --> + </div> + <!-- /.container --> + </nav> +</div> + + + <div class="container"> + + + +<!--<div class="hero-unit Spark Structured Streaming MQTT"> + <h1></h1> +</div> +--> + +<div class="row"> + <div class="col-md-12"> + <!-- + +--> + +<p>A library for writing and reading data from MQTT Servers using Spark SQL Streaming (or Structured streaming).</p> + +<h2 id="linking">Linking</h2> + +<p>Using SBT:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>libraryDependencies += "org.apache.bahir" %% "spark-sql-streaming-mqtt" % "2.3.0" +</code></pre> +</div> + +<p>Using Maven:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code><dependency> + <groupId>org.apache.bahir</groupId> + <artifactId>spark-sql-streaming-mqtt_2.11</artifactId> + <version>2.3.0</version> +</dependency> +</code></pre> +</div> + +<p>This library can also be added to Spark jobs launched through <code class="highlighter-rouge">spark-shell</code> or <code class="highlighter-rouge">spark-submit</code> by using the <code class="highlighter-rouge">--packages</code> command line option. +For example, to include it when starting the spark shell:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>$ bin/spark-shell --packages org.apache.bahir:spark-sql-streaming-mqtt_2.11:2.3.0 +</code></pre> +</div> + +<p>Unlike using <code class="highlighter-rouge">--jars</code>, using <code class="highlighter-rouge">--packages</code> ensures that this library and its dependencies will be added to the classpath. +The <code class="highlighter-rouge">--packages</code> argument can also be used with <code class="highlighter-rouge">bin/spark-submit</code>.</p> + +<p>This library is compiled for Scala 2.11 only, and intends to support Spark 2.0 onwards.</p> + +<h2 id="examples">Examples</h2> + +<p>SQL Stream can be created with data streams received through MQTT Server using:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>sqlContext.readStream + .format("org.apache.bahir.sql.streaming.mqtt.MQTTStreamSourceProvider") + .option("topic", "mytopic") + .load("tcp://localhost:1883") +</code></pre> +</div> + +<p>SQL Stream may be also transferred into MQTT messages using:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>sqlContext.writeStream + .format("org.apache.bahir.sql.streaming.mqtt.MQTTStreamSinkProvider") + .option("checkpointLocation", "/path/to/localdir") + .outputMode("complete") + .option("topic", "mytopic") + .load("tcp://localhost:1883") +</code></pre> +</div> + +<h2 id="source-recovering-from-failures">Source recovering from failures</h2> + +<p>Setting values for option <code class="highlighter-rouge">localStorage</code> and <code class="highlighter-rouge">clientId</code> helps in recovering in case of source restart, by restoring the state where it left off before the shutdown.</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>sqlContext.readStream + .format("org.apache.bahir.sql.streaming.mqtt.MQTTStreamSourceProvider") + .option("topic", "mytopic") + .option("localStorage", "/path/to/localdir") + .option("clientId", "some-client-id") + .load("tcp://localhost:1883") +</code></pre> +</div> + +<h2 id="configuration-options">Configuration options</h2> + +<p>This connector uses <a href="https://eclipse.org/paho/clients/java/";>Eclipse Paho Java Client</a>. Client API documentation is located <a href="http://www.eclipse.org/paho/files/javadoc/index.html";>here</a>.</p> + +<ul> + <li><code class="highlighter-rouge">brokerUrl</code> An URL MqttClient connects to. Set this or <code class="highlighter-rouge">path</code> as the URL of the Mqtt Server. e.g. tcp://localhost:1883.</li> + <li><code class="highlighter-rouge">persistence</code> By default it is used for storing incoming messages on disk. If <code class="highlighter-rouge">memory</code> is provided as value for this option, then recovery on restart is not supported.</li> + <li><code class="highlighter-rouge">topic</code> Topic MqttClient subscribes to.</li> + <li><code class="highlighter-rouge">clientId</code> clientId, this client is associated with. Provide the same value to recover a stopped source client. MQTT sink ignores client identifier, because Spark batch can be distributed across multiple workers whereas MQTT broker does not allow simultanous connections with same ID from multiple hosts.</li> + <li><code class="highlighter-rouge">QoS</code> The maximum quality of service to subscribe each topic at. Messages published at a lower quality of service will be received at the published QoS. Messages published at a higher quality of service will be received using the QoS specified on the subscribe.</li> + <li><code class="highlighter-rouge">username</code> Sets the user name to use for the connection to Mqtt Server. Do not set it, if server does not need this. Setting it empty will lead to errors.</li> + <li><code class="highlighter-rouge">password</code> Sets the password to use for the connection.</li> + <li><code class="highlighter-rouge">cleanSession</code> Setting it true starts a clean session, removes all checkpointed messages by a previous run of this source. This is set to false by default.</li> + <li><code class="highlighter-rouge">connectionTimeout</code> Sets the connection timeout, a value of 0 is interpretted as wait until client connects. See <code class="highlighter-rouge">MqttConnectOptions.setConnectionTimeout</code> for more information.</li> + <li><code class="highlighter-rouge">keepAlive</code> Same as <code class="highlighter-rouge">MqttConnectOptions.setKeepAliveInterval</code>.</li> + <li><code class="highlighter-rouge">mqttVersion</code> Same as <code class="highlighter-rouge">MqttConnectOptions.setMqttVersion</code>.</li> + <li><code class="highlighter-rouge">maxInflight</code> Same as <code class="highlighter-rouge">MqttConnectOptions.setMaxInflight</code></li> + <li><code class="highlighter-rouge">autoReconnect</code> Same as <code class="highlighter-rouge">MqttConnectOptions.setAutomaticReconnect</code></li> +</ul> + +<h2 id="environment-variables">Environment variables</h2> + +<p>Custom environment variables allowing to manage MQTT connectivity performed by sink connector:</p> + +<ul> + <li><code class="highlighter-rouge">spark.mqtt.client.connect.attempts</code> Number of attempts sink will try to connect to MQTT broker before failing.</li> + <li><code class="highlighter-rouge">spark.mqtt.client.connect.backoff</code> Delay in milliseconds to wait before retrying connection to the server.</li> + <li><code class="highlighter-rouge">spark.mqtt.connection.cache.timeout</code> Sink connector caches MQTT connections. Idle connections will be closed after timeout milliseconds.</li> + <li><code class="highlighter-rouge">spark.mqtt.client.publish.attempts</code> Number of attempts to publish the message before failing the task.</li> + <li><code class="highlighter-rouge">spark.mqtt.client.publish.backoff</code> Delay in milliseconds to wait before retrying send operation.</li> +</ul> + +<h3 id="scala-api">Scala API</h3> + +<p>An example, for scala API to count words from incoming message stream.</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>// Create DataFrame representing the stream of input lines from connection to mqtt server +val lines = spark.readStream + .format("org.apache.bahir.sql.streaming.mqtt.MQTTStreamSourceProvider") + .option("topic", topic) + .load(brokerUrl).selectExpr("CAST(payload AS STRING)").as[String] + +// Split the lines into words +val words = lines.map(_._1).flatMap(_.split(" ")) + +// Generate running word count +val wordCounts = words.groupBy("value").count() + +// Start running the query that prints the running counts to the console +val query = wordCounts.writeStream + .outputMode("complete") + .format("console") + .start() + +query.awaitTermination() +</code></pre> +</div> + +<p>Please see <code class="highlighter-rouge">MQTTStreamWordCount.scala</code> for full example. Review <code class="highlighter-rouge">MQTTSinkWordCount.scala</code>, if interested in publishing data to MQTT broker.</p> + +<h3 id="java-api">Java API</h3> + +<p>An example, for Java API to count words from incoming message stream.</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>// Create DataFrame representing the stream of input lines from connection to mqtt server. +Dataset<String> lines = spark + .readStream() + .format("org.apache.bahir.sql.streaming.mqtt.MQTTStreamSourceProvider") + .option("topic", topic) + .load(brokerUrl) + .selectExpr("CAST(payload AS STRING)").as(Encoders.STRING()); + +// Split the lines into words +Dataset<String> words = lines.flatMap(new FlatMapFunction<String, String>() { + @Override + public Iterator<String> call(String x) { + return Arrays.asList(x.split(" ")).iterator(); + } +}, Encoders.STRING()); + +// Generate running word count +Dataset<Row> wordCounts = words.groupBy("value").count(); + +// Start running the query that prints the running counts to the console +StreamingQuery query = wordCounts.writeStream() + .outputMode("complete") + .format("console") + .start(); + +query.awaitTermination(); +</code></pre> +</div> + +<p>Please see <code class="highlighter-rouge">JavaMQTTStreamWordCount.java</code> for full example. Review <code class="highlighter-rouge">JavaMQTTSinkWordCount.java</code>, if interested in publishing data to MQTT broker.</p> + +<h2 id="best-practices">Best Practices.</h2> + +<ol> + <li>Turn Mqtt into a more reliable messaging service.</li> +</ol> + +<blockquote> + <p><em>MQTT is a machine-to-machine (M2M)/âInternet of Thingsâ connectivity protocol. It was designed as an extremely lightweight publish/subscribe messaging transport.</em></p> +</blockquote> + +<p>The design of Mqtt and the purpose it serves goes well together, but often in an application it is of utmost value to have reliability. Since mqtt is not a distributed message queue and thus does not offer the highest level of reliability features. It should be redirected via a kafka message queue to take advantage of a distributed message queue. In fact, using a kafka message queue offers a lot of possibilities including a single kafka topic subscribed to several mqtt sources and even a single mqtt stream publishing to multiple kafka topics. Kafka is a reliable and scalable message queue.</p> + +<ol> + <li>Often the message payload is not of the default character encoding or contains binary that needs to be parsed using a particular parser. In such cases, spark mqtt payload should be processed using the external parser. For example:</li> +</ol> + +<ul> + <li> + <p>Scala API example: +<code class="highlighter-rouge">scala + // Create DataFrame representing the stream of binary messages + val lines = spark.readStream + .format("org.apache.bahir.sql.streaming.mqtt.MQTTStreamSourceProvider") + .option("topic", topic) + .load(brokerUrl).select("payload").as[Array[Byte]].map(externalParser(_)) +</code></p> + </li> + <li> + <p>Java API example +```java + // Create DataFrame representing the stream of binary messages + Dataset<byte[]> lines = spark + .readStream() + .format(âorg.apache.bahir.sql.streaming.mqtt.MQTTStreamSourceProviderâ) + .option(âtopicâ, topic) + .load(brokerUrl).selectExpr(âCAST(payload AS BINARY)â).as(Encoders.BINARY());</p> + + <div class="highlighter-rouge"><pre class="highlight"><code> // Split the lines into words + Dataset<String> words = lines.map(new MapFunction<byte[], String>() { + @Override + public String call(byte[] bytes) throws Exception { + return new String(bytes); // Plug in external parser here. + } + }, Encoders.STRING()).flatMap(new FlatMapFunction<String, String>() { + @Override + public Iterator<String> call(String x) { + return Arrays.asList(x.split(" ")).iterator(); + } + }, Encoders.STRING()); +</code></pre> + </div> + </li> +</ul> + +<p>```</p> + +<ol> + <li>What is the solution for a situation when there are a large number of varied mqtt sources, each with different schema and throughput characteristics.</li> +</ol> + +<p>Generally, one would create a lot of streaming pipelines to solve this problem. This would either require a very sophisticated scheduling setup or will waste a lot of resources, as it is not certain which stream is using more amount of data.</p> + +<p>The general solution is both less optimum and is more cumbersome to operate, with multiple moving parts incurs a high maintenance overall. As an alternative, in this situation, one can setup a single topic kafka-spark stream, where message from each of the varied stream contains a unique tag separating one from other streams. This way at the processing end, one can distinguish the message from one another and apply the right kind of decoding and processing. Similarly while storing, each message can be distinguished from others by a tag that distinguishes.</p> + + + </div> +</div> + + + + <hr> + + <!-- <p>© 2018 </p>--> + <footer class="site-footer"> + <div class="wrapper"> + <div class="footer-col-wrapper"> + + <div style="text-align:center;"> + + <div> + Copyright © 2016-2017 <a href="http://www.apache.org";>The Apache Software Foundation</a>. + Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version + 2.0</a>. + <br> + + Apache and the Apache Feather logo are trademarks of The Apache Software Foundation. + + </div> + </div> + </div> + </div> +</footer> + + </div> + + + + + <script type="text/javascript"> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-79140859-1', 'bahir.apache.org'); + ga('require', 'linkid', 'linkid.js'); + ga('send', 'pageview'); + +</script> + + + + <script src="/assets/themes/apache-clean/jquery/jquery-2.1.1.min.js"></script> + + <script src="/assets/themes/apache-clean/bootstrap/js/bootstrap.min.js"></script> + + + </body> +</html> +
http://git-wip-us.apache.org/repos/asf/bahir-website/blob/8f21f868/content/docs/spark/2.3.0/spark-streaming-akka/index.html ---------------------------------------------------------------------- diff --git a/content/docs/spark/2.3.0/spark-streaming-akka/index.html b/content/docs/spark/2.3.0/spark-streaming-akka/index.html new file mode 100644 index 0000000..f7233ab --- /dev/null +++ b/content/docs/spark/2.3.0/spark-streaming-akka/index.html @@ -0,0 +1,325 @@ + + +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Spark Streaming Akka</title> + <meta name="description" content="Spark Streaming Akka"> + <meta name="author" content=""> + + <!-- Enable responsive viewport --> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <!-- Le HTML5 shim, for IE6-8 support of HTML elements --> + <!--[if lt IE 9]> + <script src="http://html5shim.googlecode.com/svn/trunk/html5.js";></script> + <![endif]--> + + <!-- Le styles --> + <link href="/assets/themes/apache-clean/bootstrap/css/bootstrap.css" rel="stylesheet"> + <link href="/assets/themes/apache-clean/css/style.css?body=1" rel="stylesheet" type="text/css"> + <link href="/assets/themes/apache-clean/css/syntax.css" rel="stylesheet" type="text/css" media="screen" /> + <!-- Le fav and touch icons --> + <!-- Update these with your own images + <link rel="shortcut icon" href="images/favicon.ico"> + <link rel="apple-touch-icon" href="images/apple-touch-icon.png"> + <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png"> + <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png"> + --> + + <!-- make tables sortable by adding class tag "sortable" to table elements --> + <script src="http://www.kryogenix.org/code/browser/sorttable/sorttable.js";></script> + + + </head> + + <body> + + + +<!-- Navigation --> +<div id="nav-bar"> + <nav id="nav-container" class="navbar navbar-inverse " role="navigation"> + <div class="container"> + <!-- Brand and toggle get grouped for better mobile display --> + + <div class="navbar-header page-scroll"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand page-scroll" href="/#home">Home</a> + </div> + <!-- Collect the nav links, forms, and other content for toggling --> + <nav class="navbar-collapse collapse" role="navigation"> + <ul class="nav navbar-nav"> + + + + <li id="download"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Download<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/downloads/spark" target="_self">Bahir Spark Extensions</a></li> + + + <li><a href="/downloads/flink" target="_self">Bahir Flink Extensions</a></li> + + </ul> + + </li> + + + + + <li id="community"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Community<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/community" target="_self">Get Involved</a></li> + + + <li><a href="/contributing" target="_self">Contributing</a></li> + + + <li><a href="/contributing-extensions" target="_self">Contributing Extensions</a></li> + + + <li><a href="https://issues.apache.org/jira/browse/BAHIR"; target="_blank">Issue Tracker</a></li> + + + <li><a href="/community#source-code" target="_self">Source Code</a></li> + + + <li><a href="/community-members" target="_self">Project Committers</a></li> + + </ul> + + </li> + + + + + <li id="documentation"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Documentation<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/docs/spark/overview" target="_self">Bahir Spark Extensions</a></li> + + + <li><a href="/docs/flink/overview" target="_self">Bahir Flink Extensions</a></li> + + </ul> + + </li> + + + + + <li id="github"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">GitHub<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="https://github.com/apache/bahir"; target="_blank">Bahir Spark Extensions</a></li> + + + <li><a href="https://github.com/apache/bahir-flink"; target="_blank">Bahir Flink Extensions</a></li> + + + <li><a href="https://github.com/apache/bahir-website"; target="_blank">Bahir Website</a></li> + + </ul> + + </li> + + + + + <li id="apache"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Apache<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="http://www.apache.org/foundation/how-it-works.html"; target="_blank">Apache Software Foundation</a></li> + + + <li><a href="http://www.apache.org/licenses/"; target="_blank">Apache License</a></li> + + + <li><a href="http://www.apache.org/foundation/sponsorship"; target="_blank">Sponsorship</a></li> + + + <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> + + + <li><a href="/privacy-policy" target="_self">Privacy Policy</a></li> + + </ul> + + </li> + + + </ul> + </nav><!--/.navbar-collapse --> + <!-- /.navbar-collapse --> + </div> + <!-- /.container --> + </nav> +</div> + + + <div class="container"> + + + +<!--<div class="hero-unit Spark Streaming Akka"> + <h1></h1> +</div> +--> + +<div class="row"> + <div class="col-md-12"> + <!-- + +--> + +<p>A library for reading data from Akka Actors using Spark Streaming.</p> + +<h2 id="linking">Linking</h2> + +<p>Using SBT:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>libraryDependencies += "org.apache.bahir" %% "spark-streaming-akka" % "2.3.0" +</code></pre> +</div> + +<p>Using Maven:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code><dependency> + <groupId>org.apache.bahir</groupId> + <artifactId>spark-streaming-akka_2.11</artifactId> + <version>2.3.0</version> +</dependency> +</code></pre> +</div> + +<p>This library can also be added to Spark jobs launched through <code class="highlighter-rouge">spark-shell</code> or <code class="highlighter-rouge">spark-submit</code> by using the <code class="highlighter-rouge">--packages</code> command line option. +For example, to include it when starting the spark shell:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>$ bin/spark-shell --packages org.apache.bahir:spark-streaming-akka_2.11:2.3.0 +</code></pre> +</div> + +<p>Unlike using <code class="highlighter-rouge">--jars</code>, using <code class="highlighter-rouge">--packages</code> ensures that this library and its dependencies will be added to the classpath. +The <code class="highlighter-rouge">--packages</code> argument can also be used with <code class="highlighter-rouge">bin/spark-submit</code>.</p> + +<p>This library is cross-published for Scala 2.10 and Scala 2.11, so users should replace the proper Scala version (2.10 or 2.11) in the commands listed above.</p> + +<h2 id="examples">Examples</h2> + +<p>DStreams can be created with data streams received through Akka actors by using <code class="highlighter-rouge">AkkaUtils.createStream(ssc, actorProps, actor-name)</code>.</p> + +<h3 id="scala-api">Scala API</h3> + +<p>You need to extend <code class="highlighter-rouge">ActorReceiver</code> so as to store received data into Spark using <code class="highlighter-rouge">store(...)</code> methods. The supervisor strategy of +this actor can be configured to handle failures, etc.</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>class CustomActor extends ActorReceiver { + def receive = { + case data: String => store(data) + } +} + +// A new input stream can be created with this custom actor as +val ssc: StreamingContext = ... +val lines = AkkaUtils.createStream[String](ssc, Props[CustomActor](), "CustomReceiver") +</code></pre> +</div> + +<h3 id="java-api">Java API</h3> + +<p>You need to extend <code class="highlighter-rouge">JavaActorReceiver</code> so as to store received data into Spark using <code class="highlighter-rouge">store(...)</code> methods. The supervisor strategy of +this actor can be configured to handle failures, etc.</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>class CustomActor extends JavaActorReceiver { + @Override + public void onReceive(Object msg) throws Exception { + store((String) msg); + } +} + +// A new input stream can be created with this custom actor as +JavaStreamingContext jssc = ...; +JavaDStream<String> lines = AkkaUtils.<String>createStream(jssc, Props.create(CustomActor.class), "CustomReceiver"); +</code></pre> +</div> + +<p>See end-to-end examples at <a href="https://github.com/apache/bahir/tree/master/streaming-akka/examples";>Akka Examples</a></p> + + </div> +</div> + + + + <hr> + + <!-- <p>© 2018 </p>--> + <footer class="site-footer"> + <div class="wrapper"> + <div class="footer-col-wrapper"> + + <div style="text-align:center;"> + + <div> + Copyright © 2016-2017 <a href="http://www.apache.org";>The Apache Software Foundation</a>. + Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version + 2.0</a>. + <br> + + Apache and the Apache Feather logo are trademarks of The Apache Software Foundation. + + </div> + </div> + </div> + </div> +</footer> + + </div> + + + + + <script type="text/javascript"> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-79140859-1', 'bahir.apache.org'); + ga('require', 'linkid', 'linkid.js'); + ga('send', 'pageview'); + +</script> + + + + <script src="/assets/themes/apache-clean/jquery/jquery-2.1.1.min.js"></script> + + <script src="/assets/themes/apache-clean/bootstrap/js/bootstrap.min.js"></script> + + + </body> +</html> + http://git-wip-us.apache.org/repos/asf/bahir-website/blob/8f21f868/content/docs/spark/2.3.0/spark-streaming-mqtt/index.html ---------------------------------------------------------------------- diff --git a/content/docs/spark/2.3.0/spark-streaming-mqtt/index.html b/content/docs/spark/2.3.0/spark-streaming-mqtt/index.html new file mode 100644 index 0000000..c825f6c --- /dev/null +++ b/content/docs/spark/2.3.0/spark-streaming-mqtt/index.html @@ -0,0 +1,348 @@ + + +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Spark Structured Streaming MQTT</title> + <meta name="description" content="Spark Structured Streaming MQTT"> + <meta name="author" content=""> + + <!-- Enable responsive viewport --> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <!-- Le HTML5 shim, for IE6-8 support of HTML elements --> + <!--[if lt IE 9]> + <script src="http://html5shim.googlecode.com/svn/trunk/html5.js";></script> + <![endif]--> + + <!-- Le styles --> + <link href="/assets/themes/apache-clean/bootstrap/css/bootstrap.css" rel="stylesheet"> + <link href="/assets/themes/apache-clean/css/style.css?body=1" rel="stylesheet" type="text/css"> + <link href="/assets/themes/apache-clean/css/syntax.css" rel="stylesheet" type="text/css" media="screen" /> + <!-- Le fav and touch icons --> + <!-- Update these with your own images + <link rel="shortcut icon" href="images/favicon.ico"> + <link rel="apple-touch-icon" href="images/apple-touch-icon.png"> + <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png"> + <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png"> + --> + + <!-- make tables sortable by adding class tag "sortable" to table elements --> + <script src="http://www.kryogenix.org/code/browser/sorttable/sorttable.js";></script> + + + </head> + + <body> + + + +<!-- Navigation --> +<div id="nav-bar"> + <nav id="nav-container" class="navbar navbar-inverse " role="navigation"> + <div class="container"> + <!-- Brand and toggle get grouped for better mobile display --> + + <div class="navbar-header page-scroll"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand page-scroll" href="/#home">Home</a> + </div> + <!-- Collect the nav links, forms, and other content for toggling --> + <nav class="navbar-collapse collapse" role="navigation"> + <ul class="nav navbar-nav"> + + + + <li id="download"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Download<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/downloads/spark" target="_self">Bahir Spark Extensions</a></li> + + + <li><a href="/downloads/flink" target="_self">Bahir Flink Extensions</a></li> + + </ul> + + </li> + + + + + <li id="community"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Community<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/community" target="_self">Get Involved</a></li> + + + <li><a href="/contributing" target="_self">Contributing</a></li> + + + <li><a href="/contributing-extensions" target="_self">Contributing Extensions</a></li> + + + <li><a href="https://issues.apache.org/jira/browse/BAHIR"; target="_blank">Issue Tracker</a></li> + + + <li><a href="/community#source-code" target="_self">Source Code</a></li> + + + <li><a href="/community-members" target="_self">Project Committers</a></li> + + </ul> + + </li> + + + + + <li id="documentation"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Documentation<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/docs/spark/overview" target="_self">Bahir Spark Extensions</a></li> + + + <li><a href="/docs/flink/overview" target="_self">Bahir Flink Extensions</a></li> + + </ul> + + </li> + + + + + <li id="github"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">GitHub<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="https://github.com/apache/bahir"; target="_blank">Bahir Spark Extensions</a></li> + + + <li><a href="https://github.com/apache/bahir-flink"; target="_blank">Bahir Flink Extensions</a></li> + + + <li><a href="https://github.com/apache/bahir-website"; target="_blank">Bahir Website</a></li> + + </ul> + + </li> + + + + + <li id="apache"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Apache<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="http://www.apache.org/foundation/how-it-works.html"; target="_blank">Apache Software Foundation</a></li> + + + <li><a href="http://www.apache.org/licenses/"; target="_blank">Apache License</a></li> + + + <li><a href="http://www.apache.org/foundation/sponsorship"; target="_blank">Sponsorship</a></li> + + + <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> + + + <li><a href="/privacy-policy" target="_self">Privacy Policy</a></li> + + </ul> + + </li> + + + </ul> + </nav><!--/.navbar-collapse --> + <!-- /.navbar-collapse --> + </div> + <!-- /.container --> + </nav> +</div> + + + <div class="container"> + + + +<!--<div class="hero-unit Spark Structured Streaming MQTT"> + <h1></h1> +</div> +--> + +<div class="row"> + <div class="col-md-12"> + <!-- + +--> + +<p><a href="http://mqtt.org/";>MQTT</a> is MQTT is a machine-to-machine (M2M)/âInternet of Thingsâ connectivity protocol. It was designed as an extremely lightweight publish/subscribe messaging transport. It is useful for connections with remote locations where a small code footprint is required and/or network bandwidth is at a premium.</p> + +<h2 id="linking">Linking</h2> + +<p>Using SBT:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>libraryDependencies += "org.apache.bahir" %% "spark-streaming-mqtt" % "2.3.0" +</code></pre> +</div> + +<p>Using Maven:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code><dependency> + <groupId>org.apache.bahir</groupId> + <artifactId>spark-streaming-mqtt_2.11</artifactId> + <version>2.3.0</version> +</dependency> +</code></pre> +</div> + +<p>This library can also be added to Spark jobs launched through <code class="highlighter-rouge">spark-shell</code> or <code class="highlighter-rouge">spark-submit</code> by using the <code class="highlighter-rouge">--packages</code> command line option. +For example, to include it when starting the spark shell:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>$ bin/spark-shell --packages org.apache.bahir:spark-streaming-mqtt_2.11:2.3.0 +</code></pre> +</div> + +<p>Unlike using <code class="highlighter-rouge">--jars</code>, using <code class="highlighter-rouge">--packages</code> ensures that this library and its dependencies will be added to the classpath. +The <code class="highlighter-rouge">--packages</code> argument can also be used with <code class="highlighter-rouge">bin/spark-submit</code>.</p> + +<p>This library is cross-published for Scala 2.10 and Scala 2.11, so users should replace the proper Scala version (2.10 or 2.11) in the commands listed above.</p> + +<h2 id="configuration-options">Configuration options.</h2> + +<p>This source uses the <a href="https://eclipse.org/paho/clients/java/";>Eclipse Paho Java Client</a>. Client API documentation is located <a href="http://www.eclipse.org/paho/files/javadoc/index.html";>here</a>.</p> + +<ul> + <li><code class="highlighter-rouge">brokerUrl</code> A url MqttClient connects to. Set this as the url of the Mqtt Server. e.g. tcp://localhost:1883.</li> + <li><code class="highlighter-rouge">storageLevel</code> By default it is used for storing incoming messages on disk.</li> + <li><code class="highlighter-rouge">topic</code> Topic MqttClient subscribes to.</li> + <li><code class="highlighter-rouge">topics</code> List of topics MqttClient subscribes to.</li> + <li><code class="highlighter-rouge">clientId</code> clientId, this client is assoicated with. Provide the same value to recover a stopped client.</li> + <li><code class="highlighter-rouge">QoS</code> The maximum quality of service to subscribe each topic at. Messages published at a lower quality of service will be received at the published QoS. Messages published at a higher quality of service will be received using the QoS specified on the subscribe.</li> + <li><code class="highlighter-rouge">username</code> Sets the user name to use for the connection to Mqtt Server. Do not set it, if server does not need this. Setting it empty will lead to errors.</li> + <li><code class="highlighter-rouge">password</code> Sets the password to use for the connection.</li> + <li><code class="highlighter-rouge">cleanSession</code> Setting it true starts a clean session, removes all checkpointed messages by a previous run of this source. This is set to false by default.</li> + <li><code class="highlighter-rouge">connectionTimeout</code> Sets the connection timeout, a value of 0 is interpreted as wait until client connects. See <code class="highlighter-rouge">MqttConnectOptions.setConnectionTimeout</code> for more information.</li> + <li><code class="highlighter-rouge">keepAlive</code> Same as <code class="highlighter-rouge">MqttConnectOptions.setKeepAliveInterval</code>.</li> + <li><code class="highlighter-rouge">mqttVersion</code> Same as <code class="highlighter-rouge">MqttConnectOptions.setMqttVersion</code>.</li> +</ul> + +<h2 id="examples">Examples</h2> + +<h3 id="scala-api">Scala API</h3> + +<p>You need to extend <code class="highlighter-rouge">ActorReceiver</code> so as to store received data into Spark using <code class="highlighter-rouge">store(...)</code> methods. The supervisor strategy of +this actor can be configured to handle failures, etc.</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>val lines = MQTTUtils.createStream(ssc, brokerUrl, topic) +val lines = MQTTUtils.createPairedStream(ssc, brokerUrl, topic) +val lines = MQTTUtils.createPairedByteArrayStream(ssc, brokerUrl, topic) +</code></pre> +</div> + +<p>Additional mqtt connection options can be provided:</p> + +<pre><code class="language-Scala">val lines = MQTTUtils.createStream(ssc, brokerUrl, topic, storageLevel, clientId, username, password, cleanSession, qos, connectionTimeout, keepAliveInterval, mqttVersion) +val lines = MQTTUtils.createPairedStream(ssc, brokerUrl, topics, storageLevel, clientId, username, password, cleanSession, qos, connectionTimeout, keepAliveInterval, mqttVersion) +val lines = MQTTUtils.createPairedByteArrayStream(ssc, brokerUrl, topics, storageLevel, clientId, username, password, cleanSession, qos, connectionTimeout, keepAliveInterval, mqttVersion) +</code></pre> + +<h3 id="java-api">Java API</h3> + +<p>You need to extend <code class="highlighter-rouge">JavaActorReceiver</code> so as to store received data into Spark using <code class="highlighter-rouge">store(...)</code> methods. The supervisor strategy of +this actor can be configured to handle failures, etc.</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>JavaDStream<String> lines = MQTTUtils.createStream(jssc, brokerUrl, topic); +JavaReceiverInputDStream<Tuple2<String, String>> lines = MQTTUtils.createPairedStream(jssc, brokerUrl, topics); +JavaReceiverInputDStream<Tuple2<String, String>> lines = MQTTUtils.createPairedByteArrayStream(jssc, brokerUrl, topics); +</code></pre> +</div> + +<p>See end-to-end examples at <a href="https://github.com/apache/bahir/tree/master/streaming-mqtt/examples";>MQTT Examples</a></p> + +<h3 id="python-api">Python API</h3> + +<p>Create a DStream from a single topic.</p> + +<pre><code class="language-Python"> MQTTUtils.createStream(ssc, broker_url, topic) +</code></pre> + +<p>Create a DStream from a list of topics.</p> + +<pre><code class="language-Python"> MQTTUtils.createPairedStream(ssc, broker_url, topics) +</code></pre> + + </div> +</div> + + + + <hr> + + <!-- <p>© 2018 </p>--> + <footer class="site-footer"> + <div class="wrapper"> + <div class="footer-col-wrapper"> + + <div style="text-align:center;"> + + <div> + Copyright © 2016-2017 <a href="http://www.apache.org";>The Apache Software Foundation</a>. + Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version + 2.0</a>. + <br> + + Apache and the Apache Feather logo are trademarks of The Apache Software Foundation. + + </div> + </div> + </div> + </div> +</footer> + + </div> + + + + + <script type="text/javascript"> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-79140859-1', 'bahir.apache.org'); + ga('require', 'linkid', 'linkid.js'); + ga('send', 'pageview'); + +</script> + + + + <script src="/assets/themes/apache-clean/jquery/jquery-2.1.1.min.js"></script> + + <script src="/assets/themes/apache-clean/bootstrap/js/bootstrap.min.js"></script> + + + </body> +</html> + http://git-wip-us.apache.org/repos/asf/bahir-website/blob/8f21f868/content/docs/spark/2.3.0/spark-streaming-pubnub/index.html ---------------------------------------------------------------------- diff --git a/content/docs/spark/2.3.0/spark-streaming-pubnub/index.html b/content/docs/spark/2.3.0/spark-streaming-pubnub/index.html new file mode 100644 index 0000000..8c3bbe4 --- /dev/null +++ b/content/docs/spark/2.3.0/spark-streaming-pubnub/index.html @@ -0,0 +1,338 @@ + + +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Spark Streaming Google Pub-Sub</title> + <meta name="description" content="Spark Streaming Google Pub-Sub"> + <meta name="author" content=""> + + <!-- Enable responsive viewport --> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <!-- Le HTML5 shim, for IE6-8 support of HTML elements --> + <!--[if lt IE 9]> + <script src="http://html5shim.googlecode.com/svn/trunk/html5.js";></script> + <![endif]--> + + <!-- Le styles --> + <link href="/assets/themes/apache-clean/bootstrap/css/bootstrap.css" rel="stylesheet"> + <link href="/assets/themes/apache-clean/css/style.css?body=1" rel="stylesheet" type="text/css"> + <link href="/assets/themes/apache-clean/css/syntax.css" rel="stylesheet" type="text/css" media="screen" /> + <!-- Le fav and touch icons --> + <!-- Update these with your own images + <link rel="shortcut icon" href="images/favicon.ico"> + <link rel="apple-touch-icon" href="images/apple-touch-icon.png"> + <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png"> + <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png"> + --> + + <!-- make tables sortable by adding class tag "sortable" to table elements --> + <script src="http://www.kryogenix.org/code/browser/sorttable/sorttable.js";></script> + + + </head> + + <body> + + + +<!-- Navigation --> +<div id="nav-bar"> + <nav id="nav-container" class="navbar navbar-inverse " role="navigation"> + <div class="container"> + <!-- Brand and toggle get grouped for better mobile display --> + + <div class="navbar-header page-scroll"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand page-scroll" href="/#home">Home</a> + </div> + <!-- Collect the nav links, forms, and other content for toggling --> + <nav class="navbar-collapse collapse" role="navigation"> + <ul class="nav navbar-nav"> + + + + <li id="download"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Download<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/downloads/spark" target="_self">Bahir Spark Extensions</a></li> + + + <li><a href="/downloads/flink" target="_self">Bahir Flink Extensions</a></li> + + </ul> + + </li> + + + + + <li id="community"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Community<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/community" target="_self">Get Involved</a></li> + + + <li><a href="/contributing" target="_self">Contributing</a></li> + + + <li><a href="/contributing-extensions" target="_self">Contributing Extensions</a></li> + + + <li><a href="https://issues.apache.org/jira/browse/BAHIR"; target="_blank">Issue Tracker</a></li> + + + <li><a href="/community#source-code" target="_self">Source Code</a></li> + + + <li><a href="/community-members" target="_self">Project Committers</a></li> + + </ul> + + </li> + + + + + <li id="documentation"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Documentation<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/docs/spark/overview" target="_self">Bahir Spark Extensions</a></li> + + + <li><a href="/docs/flink/overview" target="_self">Bahir Flink Extensions</a></li> + + </ul> + + </li> + + + + + <li id="github"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">GitHub<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="https://github.com/apache/bahir"; target="_blank">Bahir Spark Extensions</a></li> + + + <li><a href="https://github.com/apache/bahir-flink"; target="_blank">Bahir Flink Extensions</a></li> + + + <li><a href="https://github.com/apache/bahir-website"; target="_blank">Bahir Website</a></li> + + </ul> + + </li> + + + + + <li id="apache"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Apache<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="http://www.apache.org/foundation/how-it-works.html"; target="_blank">Apache Software Foundation</a></li> + + + <li><a href="http://www.apache.org/licenses/"; target="_blank">Apache License</a></li> + + + <li><a href="http://www.apache.org/foundation/sponsorship"; target="_blank">Sponsorship</a></li> + + + <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> + + + <li><a href="/privacy-policy" target="_self">Privacy Policy</a></li> + + </ul> + + </li> + + + </ul> + </nav><!--/.navbar-collapse --> + <!-- /.navbar-collapse --> + </div> + <!-- /.container --> + </nav> +</div> + + + <div class="container"> + + + +<!--<div class="hero-unit Spark Streaming Google Pub-Sub"> + <h1></h1> +</div> +--> + +<div class="row"> + <div class="col-md-12"> + <!-- + +--> + +<h1 id="spark-streaming-pubnub-connector">Spark Streaming PubNub Connector</h1> + +<p>Library for reading data from real-time messaging infrastructure <a href="https://www.pubnub.com/";>PubNub</a> using Spark Streaming.</p> + +<h2 id="linking">Linking</h2> + +<p>Using SBT:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>libraryDependencies += "org.apache.bahir" %% "spark-streaming-pubnub" % "2.3.0" +</code></pre> +</div> + +<p>Using Maven:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code><dependency> + <groupId>org.apache.bahir</groupId> + <artifactId>spark-streaming-pubnub_2.11</artifactId> + <version>2.3.0</version> +</dependency> +</code></pre> +</div> + +<p>This library can also be added to Spark jobs launched through <code class="highlighter-rouge">spark-shell</code> or <code class="highlighter-rouge">spark-submit</code> by using the <code class="highlighter-rouge">--packages</code> command line option. +For example, to include it when starting the spark shell:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>$ bin/spark-shell --packages org.apache.bahir:spark-streaming-pubnub_2.11:2.3.0 +</code></pre> +</div> + +<p>Unlike using <code class="highlighter-rouge">--jars</code>, using <code class="highlighter-rouge">--packages</code> ensures that this library and its dependencies will be added to the classpath. +The <code class="highlighter-rouge">--packages</code> argument can also be used with <code class="highlighter-rouge">bin/spark-submit</code>.</p> + +<h2 id="examples">Examples</h2> + +<p>Connector leverages official Java client for PubNub cloud infrastructure. You can import the <code class="highlighter-rouge">PubNubUtils</code> +class and create input stream by calling <code class="highlighter-rouge">PubNubUtils.createStream()</code> as shown below. Security and performance related +features shall be setup inside standard <code class="highlighter-rouge">PNConfiguration</code> object. We advise to configure reconnection policy so that +temporary network outages do not interrupt processing job. Users may subscribe to multiple channels and channel groups, +as well as specify time token to start receiving messages since given point in time.</p> + +<p>For complete code examples, please review <em>examples</em> directory.</p> + +<h3 id="scala-api">Scala API</h3> + +<div class="highlighter-rouge"><pre class="highlight"><code>import com.pubnub.api.PNConfiguration +import com.pubnub.api.enums.PNReconnectionPolicy + +import org.apache.spark.streaming.pubnub.{PubNubUtils, SparkPubNubMessage} + +val config = new PNConfiguration +config.setSubscribeKey(subscribeKey) +config.setSecure(true) +config.setReconnectionPolicy(PNReconnectionPolicy.LINEAR) +val channel = "my-channel" + +val pubNubStream: ReceiverInputDStream[SparkPubNubMessage] = PubNubUtils.createStream( + ssc, config, Seq(channel), Seq(), None, StorageLevel.MEMORY_AND_DISK_SER_2 +) +</code></pre> +</div> + +<h3 id="java-api">Java API</h3> + +<div class="highlighter-rouge"><pre class="highlight"><code>import com.pubnub.api.PNConfiguration +import com.pubnub.api.enums.PNReconnectionPolicy + +import org.apache.spark.streaming.pubnub.PubNubUtils +import org.apache.spark.streaming.pubnub.SparkPubNubMessage + +PNConfiguration config = new PNConfiguration() +config.setSubscribeKey(subscribeKey) +config.setSecure(true) +config.setReconnectionPolicy(PNReconnectionPolicy.LINEAR) +Set<String> channels = new HashSet<String>() ; + +ReceiverInputDStream<SparkPubNubMessage> pubNubStream = PubNubUtils.createStream( + ssc, config, channels, Collections.EMPTY_SET, null, + StorageLevel.MEMORY_AND_DISK_SER_2() +) +</code></pre> +</div> + +<h2 id="unit-test">Unit Test</h2> + +<p>Unit tests take advantage of publicly available <em>demo</em> subscription and and publish key, which has limited request rate.</p> + + </div> +</div> + + + + <hr> + + <!-- <p>© 2018 </p>--> + <footer class="site-footer"> + <div class="wrapper"> + <div class="footer-col-wrapper"> + + <div style="text-align:center;"> + + <div> + Copyright © 2016-2017 <a href="http://www.apache.org";>The Apache Software Foundation</a>. + Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version + 2.0</a>. + <br> + + Apache and the Apache Feather logo are trademarks of The Apache Software Foundation. + + </div> + </div> + </div> + </div> +</footer> + + </div> + + + + + <script type="text/javascript"> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-79140859-1', 'bahir.apache.org'); + ga('require', 'linkid', 'linkid.js'); + ga('send', 'pageview'); + +</script> + + + + <script src="/assets/themes/apache-clean/jquery/jquery-2.1.1.min.js"></script> + + <script src="/assets/themes/apache-clean/bootstrap/js/bootstrap.min.js"></script> + + + </body> +</html> + http://git-wip-us.apache.org/repos/asf/bahir-website/blob/8f21f868/content/docs/spark/2.3.0/spark-streaming-pubsub/index.html ---------------------------------------------------------------------- diff --git a/content/docs/spark/2.3.0/spark-streaming-pubsub/index.html b/content/docs/spark/2.3.0/spark-streaming-pubsub/index.html new file mode 100644 index 0000000..299faa4 --- /dev/null +++ b/content/docs/spark/2.3.0/spark-streaming-pubsub/index.html @@ -0,0 +1,335 @@ + + +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Spark Streaming PubNub</title> + <meta name="description" content="Spark Streaming PubNub"> + <meta name="author" content=""> + + <!-- Enable responsive viewport --> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <!-- Le HTML5 shim, for IE6-8 support of HTML elements --> + <!--[if lt IE 9]> + <script src="http://html5shim.googlecode.com/svn/trunk/html5.js";></script> + <![endif]--> + + <!-- Le styles --> + <link href="/assets/themes/apache-clean/bootstrap/css/bootstrap.css" rel="stylesheet"> + <link href="/assets/themes/apache-clean/css/style.css?body=1" rel="stylesheet" type="text/css"> + <link href="/assets/themes/apache-clean/css/syntax.css" rel="stylesheet" type="text/css" media="screen" /> + <!-- Le fav and touch icons --> + <!-- Update these with your own images + <link rel="shortcut icon" href="images/favicon.ico"> + <link rel="apple-touch-icon" href="images/apple-touch-icon.png"> + <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png"> + <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png"> + --> + + <!-- make tables sortable by adding class tag "sortable" to table elements --> + <script src="http://www.kryogenix.org/code/browser/sorttable/sorttable.js";></script> + + + </head> + + <body> + + + +<!-- Navigation --> +<div id="nav-bar"> + <nav id="nav-container" class="navbar navbar-inverse " role="navigation"> + <div class="container"> + <!-- Brand and toggle get grouped for better mobile display --> + + <div class="navbar-header page-scroll"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand page-scroll" href="/#home">Home</a> + </div> + <!-- Collect the nav links, forms, and other content for toggling --> + <nav class="navbar-collapse collapse" role="navigation"> + <ul class="nav navbar-nav"> + + + + <li id="download"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Download<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/downloads/spark" target="_self">Bahir Spark Extensions</a></li> + + + <li><a href="/downloads/flink" target="_self">Bahir Flink Extensions</a></li> + + </ul> + + </li> + + + + + <li id="community"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Community<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/community" target="_self">Get Involved</a></li> + + + <li><a href="/contributing" target="_self">Contributing</a></li> + + + <li><a href="/contributing-extensions" target="_self">Contributing Extensions</a></li> + + + <li><a href="https://issues.apache.org/jira/browse/BAHIR"; target="_blank">Issue Tracker</a></li> + + + <li><a href="/community#source-code" target="_self">Source Code</a></li> + + + <li><a href="/community-members" target="_self">Project Committers</a></li> + + </ul> + + </li> + + + + + <li id="documentation"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Documentation<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/docs/spark/overview" target="_self">Bahir Spark Extensions</a></li> + + + <li><a href="/docs/flink/overview" target="_self">Bahir Flink Extensions</a></li> + + </ul> + + </li> + + + + + <li id="github"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">GitHub<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="https://github.com/apache/bahir"; target="_blank">Bahir Spark Extensions</a></li> + + + <li><a href="https://github.com/apache/bahir-flink"; target="_blank">Bahir Flink Extensions</a></li> + + + <li><a href="https://github.com/apache/bahir-website"; target="_blank">Bahir Website</a></li> + + </ul> + + </li> + + + + + <li id="apache"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Apache<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="http://www.apache.org/foundation/how-it-works.html"; target="_blank">Apache Software Foundation</a></li> + + + <li><a href="http://www.apache.org/licenses/"; target="_blank">Apache License</a></li> + + + <li><a href="http://www.apache.org/foundation/sponsorship"; target="_blank">Sponsorship</a></li> + + + <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> + + + <li><a href="/privacy-policy" target="_self">Privacy Policy</a></li> + + </ul> + + </li> + + + </ul> + </nav><!--/.navbar-collapse --> + <!-- /.navbar-collapse --> + </div> + <!-- /.container --> + </nav> +</div> + + + <div class="container"> + + + +<!--<div class="hero-unit Spark Streaming PubNub"> + <h1></h1> +</div> +--> + +<div class="row"> + <div class="col-md-12"> + <!-- + +--> + +<p>A library for reading data from <a href="https://cloud.google.com/pubsub/";>Google Cloud Pub/Sub</a> using Spark Streaming.</p> + +<h2 id="linking">Linking</h2> + +<p>Using SBT:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>libraryDependencies += "org.apache.bahir" %% "spark-streaming-pubsub" % "2.3.0" +</code></pre> +</div> + +<p>Using Maven:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code><dependency> + <groupId>org.apache.bahir</groupId> + <artifactId>spark-streaming-pubsub_2.11</artifactId> + <version>2.3.0</version> +</dependency> +</code></pre> +</div> + +<p>This library can also be added to Spark jobs launched through <code class="highlighter-rouge">spark-shell</code> or <code class="highlighter-rouge">spark-submit</code> by using the <code class="highlighter-rouge">--packages</code> command line option. +For example, to include it when starting the spark shell:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>$ bin/spark-shell --packages org.apache.bahir:spark-streaming-pubsub_2.11:2.3.0 +</code></pre> +</div> + +<p>Unlike using <code class="highlighter-rouge">--jars</code>, using <code class="highlighter-rouge">--packages</code> ensures that this library and its dependencies will be added to the classpath. +The <code class="highlighter-rouge">--packages</code> argument can also be used with <code class="highlighter-rouge">bin/spark-submit</code>.</p> + +<h2 id="examples">Examples</h2> + +<p>First you need to create credential by SparkGCPCredentials, it support four type of credentials +* application default + <code class="highlighter-rouge">SparkGCPCredentials.builder.build()</code> +* json type service account + <code class="highlighter-rouge">SparkGCPCredentials.builder.jsonServiceAccount(PATH_TO_JSON_KEY).build()</code> +* p12 type service account + <code class="highlighter-rouge">SparkGCPCredentials.builder.p12ServiceAccount(PATH_TO_P12_KEY, EMAIL_ACCOUNT).build()</code> +* metadata service account(running on dataproc) + <code class="highlighter-rouge">SparkGCPCredentials.builder.metadataServiceAccount().build()</code></p> + +<h3 id="scala-api">Scala API</h3> + +<div class="highlighter-rouge"><pre class="highlight"><code>val lines = PubsubUtils.createStream(ssc, projectId, subscriptionName, credential, ..) +</code></pre> +</div> + +<h3 id="java-api">Java API</h3> + +<div class="highlighter-rouge"><pre class="highlight"><code>JavaDStream<SparkPubsubMessage> lines = PubsubUtils.createStream(jssc, projectId, subscriptionName, credential...) +</code></pre> +</div> + +<p>See end-to-end examples at <a href="streaming-pubsub/examples">Google Cloud Pubsub Examples</a></p> + +<h3 id="unit-test">Unit Test</h3> + +<p>To run the PubSub test cases, you need to generate <strong>Google API service account key files</strong> and set the corresponding environment variable to enable the test.</p> + +<h4 id="to-generate-a-service-account-key-file-with-pubsub-permission">To generate a service account key file with PubSub permission</h4> + +<ol> + <li>Go to <a href="console.cloud.google.com">Google API Console</a></li> + <li>Choose the <code class="highlighter-rouge">Credentials</code> Tab> <code class="highlighter-rouge">Create credentials</code> button> <code class="highlighter-rouge">Service account key</code></li> + <li>Fill the account name, assign <code class="highlighter-rouge">Role> Pub/Sub> Pub/Sub Editor</code> and check the option <code class="highlighter-rouge">Furnish a private key</code> to create one. You need to create one for JSON key file, another for P12.</li> + <li>The account email is the <code class="highlighter-rouge">Service account ID</code></li> +</ol> + +<h4 id="setting-the-environment-variables-and-run-test">Setting the environment variables and run test</h4> + +<div class="highlighter-rouge"><pre class="highlight"><code>mvn clean package -DskipTests -pl streaming-pubsub + +export ENABLE_PUBSUB_TESTS=1 +export GCP_TEST_ACCOUNT="THE_P12_SERVICE_ACCOUNT_ID_MENTIONED_ABOVE" +export GCP_TEST_PROJECT_ID="YOUR_GCP_PROJECT_ID" +export GCP_TEST_JSON_KEY_PATH=/path/to/pubsub/credential/files/Apache-Bahir-PubSub-1234abcd.json +export GCP_TEST_P12_KEY_PATH=/path/to/pubsub/credential/files/Apache-Bahir-PubSub-5678efgh.p12 + +mvn test -pl streaming-pubsub +</code></pre> +</div> + + </div> +</div> + + + + <hr> + + <!-- <p>© 2018 </p>--> + <footer class="site-footer"> + <div class="wrapper"> + <div class="footer-col-wrapper"> + + <div style="text-align:center;"> + + <div> + Copyright © 2016-2017 <a href="http://www.apache.org";>The Apache Software Foundation</a>. + Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version + 2.0</a>. + <br> + + Apache and the Apache Feather logo are trademarks of The Apache Software Foundation. + + </div> + </div> + </div> + </div> +</footer> + + </div> + + + + + <script type="text/javascript"> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-79140859-1', 'bahir.apache.org'); + ga('require', 'linkid', 'linkid.js'); + ga('send', 'pageview'); + +</script> + + + + <script src="/assets/themes/apache-clean/jquery/jquery-2.1.1.min.js"></script> + + <script src="/assets/themes/apache-clean/bootstrap/js/bootstrap.min.js"></script> + + + </body> +</html> + http://git-wip-us.apache.org/repos/asf/bahir-website/blob/8f21f868/content/docs/spark/2.3.0/spark-streaming-twitter/index.html ---------------------------------------------------------------------- diff --git a/content/docs/spark/2.3.0/spark-streaming-twitter/index.html b/content/docs/spark/2.3.0/spark-streaming-twitter/index.html new file mode 100644 index 0000000..d853682 --- /dev/null +++ b/content/docs/spark/2.3.0/spark-streaming-twitter/index.html @@ -0,0 +1,308 @@ + + +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Spark Streaming Twitter</title> + <meta name="description" content="Spark Streaming Twitter"> + <meta name="author" content=""> + + <!-- Enable responsive viewport --> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <!-- Le HTML5 shim, for IE6-8 support of HTML elements --> + <!--[if lt IE 9]> + <script src="http://html5shim.googlecode.com/svn/trunk/html5.js";></script> + <![endif]--> + + <!-- Le styles --> + <link href="/assets/themes/apache-clean/bootstrap/css/bootstrap.css" rel="stylesheet"> + <link href="/assets/themes/apache-clean/css/style.css?body=1" rel="stylesheet" type="text/css"> + <link href="/assets/themes/apache-clean/css/syntax.css" rel="stylesheet" type="text/css" media="screen" /> + <!-- Le fav and touch icons --> + <!-- Update these with your own images + <link rel="shortcut icon" href="images/favicon.ico"> + <link rel="apple-touch-icon" href="images/apple-touch-icon.png"> + <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png"> + <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png"> + --> + + <!-- make tables sortable by adding class tag "sortable" to table elements --> + <script src="http://www.kryogenix.org/code/browser/sorttable/sorttable.js";></script> + + + </head> + + <body> + + + +<!-- Navigation --> +<div id="nav-bar"> + <nav id="nav-container" class="navbar navbar-inverse " role="navigation"> + <div class="container"> + <!-- Brand and toggle get grouped for better mobile display --> + + <div class="navbar-header page-scroll"> + <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand page-scroll" href="/#home">Home</a> + </div> + <!-- Collect the nav links, forms, and other content for toggling --> + <nav class="navbar-collapse collapse" role="navigation"> + <ul class="nav navbar-nav"> + + + + <li id="download"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Download<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/downloads/spark" target="_self">Bahir Spark Extensions</a></li> + + + <li><a href="/downloads/flink" target="_self">Bahir Flink Extensions</a></li> + + </ul> + + </li> + + + + + <li id="community"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Community<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/community" target="_self">Get Involved</a></li> + + + <li><a href="/contributing" target="_self">Contributing</a></li> + + + <li><a href="/contributing-extensions" target="_self">Contributing Extensions</a></li> + + + <li><a href="https://issues.apache.org/jira/browse/BAHIR"; target="_blank">Issue Tracker</a></li> + + + <li><a href="/community#source-code" target="_self">Source Code</a></li> + + + <li><a href="/community-members" target="_self">Project Committers</a></li> + + </ul> + + </li> + + + + + <li id="documentation"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Documentation<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="/docs/spark/overview" target="_self">Bahir Spark Extensions</a></li> + + + <li><a href="/docs/flink/overview" target="_self">Bahir Flink Extensions</a></li> + + </ul> + + </li> + + + + + <li id="github"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">GitHub<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="https://github.com/apache/bahir"; target="_blank">Bahir Spark Extensions</a></li> + + + <li><a href="https://github.com/apache/bahir-flink"; target="_blank">Bahir Flink Extensions</a></li> + + + <li><a href="https://github.com/apache/bahir-website"; target="_blank">Bahir Website</a></li> + + </ul> + + </li> + + + + + <li id="apache"> + + <a href="#" data-toggle="dropdown" class="dropdown-toggle">Apache<b class="caret"></b></a> + <ul class="dropdown-menu dropdown-left"> + + + <li><a href="http://www.apache.org/foundation/how-it-works.html"; target="_blank">Apache Software Foundation</a></li> + + + <li><a href="http://www.apache.org/licenses/"; target="_blank">Apache License</a></li> + + + <li><a href="http://www.apache.org/foundation/sponsorship"; target="_blank">Sponsorship</a></li> + + + <li><a href="http://www.apache.org/foundation/thanks.html"; target="_blank">Thanks</a></li> + + + <li><a href="/privacy-policy" target="_self">Privacy Policy</a></li> + + </ul> + + </li> + + + </ul> + </nav><!--/.navbar-collapse --> + <!-- /.navbar-collapse --> + </div> + <!-- /.container --> + </nav> +</div> + + + <div class="container"> + + + +<!--<div class="hero-unit Spark Streaming Twitter"> + <h1></h1> +</div> +--> + +<div class="row"> + <div class="col-md-12"> + <!-- + +--> + +<p>A library for reading social data from <a href="http://twitter.com/";>twitter</a> using Spark Streaming.</p> + +<h2 id="linking">Linking</h2> + +<p>Using SBT:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>libraryDependencies += "org.apache.bahir" %% "spark-streaming-twitter" % "2.3.0" +</code></pre> +</div> + +<p>Using Maven:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code><dependency> + <groupId>org.apache.bahir</groupId> + <artifactId>spark-streaming-twitter_2.11</artifactId> + <version>2.3.0</version> +</dependency> +</code></pre> +</div> + +<p>This library can also be added to Spark jobs launched through <code class="highlighter-rouge">spark-shell</code> or <code class="highlighter-rouge">spark-submit</code> by using the <code class="highlighter-rouge">--packages</code> command line option. +For example, to include it when starting the spark shell:</p> + +<div class="highlighter-rouge"><pre class="highlight"><code>$ bin/spark-shell --packages org.apache.bahir:spark-streaming-twitter_2.11:2.3.0 +</code></pre> +</div> + +<p>Unlike using <code class="highlighter-rouge">--jars</code>, using <code class="highlighter-rouge">--packages</code> ensures that this library and its dependencies will be added to the classpath. +The <code class="highlighter-rouge">--packages</code> argument can also be used with <code class="highlighter-rouge">bin/spark-submit</code>.</p> + +<p>This library is cross-published for Scala 2.10 and Scala 2.11, so users should replace the proper Scala version (2.10 or 2.11) in the commands listed above.</p> + +<h2 id="examples">Examples</h2> + +<p><code class="highlighter-rouge">TwitterUtils</code> uses Twitter4j to get the public stream of tweets using <a href="https://dev.twitter.com/docs/streaming-apis";>Twitterâs Streaming API</a>. Authentication information +can be provided by any of the <a href="http://twitter4j.org/en/configuration.html";>methods</a> supported by Twitter4J library. You can import the <code class="highlighter-rouge">TwitterUtils</code> class and create a DStream with <code class="highlighter-rouge">TwitterUtils.createStream</code> as shown below.</p> + +<h3 id="scala-api">Scala API</h3> + +<div class="highlighter-rouge"><pre class="highlight"><code>import org.apache.spark.streaming.twitter._ + +TwitterUtils.createStream(ssc, None) +</code></pre> +</div> + +<h3 id="java-api">Java API</h3> + +<div class="highlighter-rouge"><pre class="highlight"><code>import org.apache.spark.streaming.twitter.*; + +TwitterUtils.createStream(jssc); +</code></pre> +</div> + +<p>You can also either get the public stream, or get the filtered stream based on keywords. +See end-to-end examples at <a href="https://github.com/apache/bahir/tree/master/streaming-twitter/examples";>Twitter Examples</a></p> + + </div> +</div> + + + + <hr> + + <!-- <p>© 2018 </p>--> + <footer class="site-footer"> + <div class="wrapper"> + <div class="footer-col-wrapper"> + + <div style="text-align:center;"> + + <div> + Copyright © 2016-2017 <a href="http://www.apache.org";>The Apache Software Foundation</a>. + Licensed under the <a href="http://www.apache.org/licenses/LICENSE-2.0";>Apache License, Version + 2.0</a>. + <br> + + Apache and the Apache Feather logo are trademarks of The Apache Software Foundation. + + </div> + </div> + </div> + </div> +</footer> + + </div> + + + + + <script type="text/javascript"> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-79140859-1', 'bahir.apache.org'); + ga('require', 'linkid', 'linkid.js'); + ga('send', 'pageview'); + +</script> + + + + <script src="/assets/themes/apache-clean/jquery/jquery-2.1.1.min.js"></script> + + <script src="/assets/themes/apache-clean/bootstrap/js/bootstrap.min.js"></script> + + + </body> +</html> +
