all files needed for decent maven site
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/commit/107852b2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/tree/107852b2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/diff/107852b2 Branch: refs/heads/0.2-incubating-rc1 Commit: 107852b258d6a6d31ba9e9447105a1a2f88f0aed Parents: a1836d0 Author: Steve Blackmon <sblack...@apache.org> Authored: Mon Apr 27 22:25:14 2015 -0500 Committer: Steve Blackmon <sblack...@apache.org> Committed: Mon Apr 27 22:25:14 2015 -0500 ---------------------------------------------------------------------- local/elasticsearch-hdfs/ElasticsearchHdfs.md | 47 ---- local/elasticsearch-hdfs/HdfsElasticsearch.md | 50 ---- local/elasticsearch-hdfs/README.md | 4 +- local/elasticsearch-hdfs/pom.xml | 19 +- .../main/resources/ElasticsearchHdfs.dot.png | Bin 0 -> 23205 bytes .../main/resources/HdfsElasticsearch.dot.png | Bin 0 -> 23439 bytes .../src/site/markdown/ElasticsearchHdfs.md | 47 ++++ .../src/site/markdown/HdfsElasticsearch.md | 50 ++++ .../src/site/markdown/index.md | 35 +++ .../src/site/resources/ElasticsearchHdfs.dot | 17 ++ .../site/resources/ElasticsearchHdfs.dot.png | Bin 0 -> 23205 bytes .../ElasticsearchHdfsConfiguration.json | 10 + .../src/site/resources/HdfsElasticsearch.dot | 17 ++ .../site/resources/HdfsElasticsearch.dot.png | Bin 0 -> 23439 bytes .../HdfsElasticsearchConfiguration.json | 10 + .../src/site/resources/testBackup.json | 24 ++ .../src/site/resources/testRestore.json | 20 ++ .../ElasticsearchReindex.png | Bin 8913 -> 0 bytes local/elasticsearch-reindex/README.md | 4 +- local/elasticsearch-reindex/pom.xml | 18 +- .../main/resources/ElasticsearchReindex.dot.png | Bin 0 -> 23187 bytes .../src/site/markdown/index.md | 79 +++++++ .../resources/ActivityChildObjectParent.json | 15 ++ .../src/site/resources/ElasticsearchReindex.dot | 17 ++ .../site/resources/ElasticsearchReindex.dot.png | Bin 0 -> 23187 bytes .../ElasticsearchReindexConfiguration.json | 10 + .../src/site/resources/application.json | 26 ++ .../src/site/resources/testReindex.json | 26 ++ .../src/site/resources/testReindexChild.json | 26 ++ .../src/site/resources/testReindexParent.json | 26 ++ .../MongoElasticsearchSync.png | Bin 11502 -> 0 bytes local/mongo-elasticsearch-sync/README.md | 4 +- local/mongo-elasticsearch-sync/pom.xml | 3 + .../resources/MongoElasticsearchSync.dot.png | Bin 0 -> 27856 bytes .../src/site/markdown/index.md | 68 ++++++ .../site/resources/MongoElasticsearchSync.dot | 18 ++ .../resources/MongoElasticsearchSync.dot.png | Bin 0 -> 27856 bytes .../MongoElasticsearchSyncConfiguration.json | 10 + .../src/site/resources/application.json | 17 ++ .../src/site/resources/testSync.json | 18 ++ local/twitter-follow-graph/README.md | 2 +- .../twitter-follow-graph/TwitterFollowGraph.png | Bin 35826 -> 0 bytes local/twitter-follow-graph/pom.xml | 21 +- .../main/resources/TwitterFollowGraph.dot.png | Bin 0 -> 34724 bytes .../src/site/markdown/index.md | 97 ++++++++ .../src/site/resources/TwitterFollowGraph.dot | 21 ++ .../site/resources/TwitterFollowGraph.dot.png | Bin 0 -> 34724 bytes .../TwitterFollowGraphConfiguration.json | 13 + local/twitter-history-elasticsearch/README.md | 2 +- .../TwitterHistoryElasticsearch.png | Bin 27497 -> 0 bytes local/twitter-history-elasticsearch/pom.xml | 14 +- .../TwitterHistoryElasticsearch.dot.png | Bin 0 -> 26774 bytes .../src/site/markdown/index.md | 78 ++++++ .../resources/TwitterHistoryElasticsearch.dot | 21 ++ .../TwitterHistoryElasticsearch.dot.png | Bin 0 -> 26774 bytes .../src/site/resources/application.conf | 23 ++ ...witterHistoryElasticsearchConfiguration.json | 13 + .../twitter-userstream-elasticsearch/README.md | 4 +- .../TwitterUserstreamElasticsearch.png | Bin 65713 -> 0 bytes local/twitter-userstream-elasticsearch/pom.xml | 12 +- .../TwitterUserstreamElasticsearch.dot.png | Bin 0 -> 64469 bytes .../src/site/markdown/index.md | 73 ++++++ .../TwitterUserstreamElasticsearch.dot | 28 +++ .../TwitterUserstreamElasticsearch.dot.png | Bin 0 -> 64469 bytes .../src/site/resources/application.conf | 22 ++ .../twitter/TwitterUserstreamElasticsearch.json | 14 ++ pom.xml | 235 ++++++++++++++++++- src/site/site.xml | 29 +++ 68 files changed, 1293 insertions(+), 164 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/ElasticsearchHdfs.md ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/ElasticsearchHdfs.md b/local/elasticsearch-hdfs/ElasticsearchHdfs.md deleted file mode 100644 index b42d1a8..0000000 --- a/local/elasticsearch-hdfs/ElasticsearchHdfs.md +++ /dev/null @@ -1,47 +0,0 @@ -Apache Streams (incubating) -Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0 --------------------------------------------------------------------------------- - -elasticsearch-hdfs -============================== - -Description: ------------------ - -Copies documents from elasticsearch to hdfs. - -Specification: ------------------ - -[ElasticsearchHdfs.dot](src/main/resources/ElasticsearchHdfs.dot "ElasticsearchHdfs.dot" ) - -Diagram: ------------------ - -![ElasticsearchHdfs.png](./ElasticsearchHdfs.png?raw=true) - -Example Configuration: ----------------------- - - { - "source": { - "host": "localhost", - "port": 27017, - "db": "streams", - "collection": "activities" - }, - "destination": { - "hosts": [ - "localhost" - ], - "port": 9300, - "clusterName": "elasticsearch", - "index": "destination", - "type": "activity" - } - } - -Run: --------- - -`docker run elasticsearch-hdfs:0.2-incubating-SNAPSHOT-PP.jar java -cp stash-migrate-0.2-incubating-SNAPSHOT.jar -Dconfig.file=http://<location_of_config_file>.json org.apache.streams.elasticsearch.example.HdfsElasticsearch` http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/HdfsElasticsearch.md ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/HdfsElasticsearch.md b/local/elasticsearch-hdfs/HdfsElasticsearch.md deleted file mode 100644 index c4ca261..0000000 --- a/local/elasticsearch-hdfs/HdfsElasticsearch.md +++ /dev/null @@ -1,50 +0,0 @@ -Apache Streams (incubating) -Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0 --------------------------------------------------------------------------------- - -hdfs-elasticsearch -============================== - -Description: ------------------ - -Copies documents from hdfs to elasticsearch. - -Specification: ------------------ - -[HdfsElasticsearch.dot](src/main/resources/HdfsElasticsearch.dot "HdfsElasticsearch.dot" ) - -Diagram: ------------------ - -![HdfsElasticsearch.png](./HdfsElasticsearch.png?raw=true) - -Example Configuration: ----------------------- - - { - "source": { - "scheme": "file", - "host": "localhost", - "user": "cloudera", - "path": "/tmp", - "writerPath": "activity" - }, - "destination": { - "hosts": [ - "localhost" - ], - "port": 9300, - "clusterName": "elasticsearch", - "index": "activity2", - "type": "activity", - "forceUseConfig": true - } - } - - -Run: --------- - -`docker run elasticsearch-hdfs:0.2-incubating-SNAPSHOT.jar java -cp elasticsearch-hdfs-0.2-incubating-SNAPSHOT.jar -Dconfig.file=file://<location_of_config_file>.json org.apache.streams.elasticsearch.example.HdfsElasticsearch` http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/README.md ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/README.md b/local/elasticsearch-hdfs/README.md index 349046d..b3e2e3f 100644 --- a/local/elasticsearch-hdfs/README.md +++ b/local/elasticsearch-hdfs/README.md @@ -16,9 +16,9 @@ Copies documents between elasticsearch and file system using the hdfs persist mo Streams: -------- -[ElasticsearchHdfs](ElasticsearchHdfs.md "ElasticsearchHdfs" ) +[ElasticsearchHdfs](src/site/markdown/ElasticsearchHdfs.md "ElasticsearchHdfs" ) -[HdfsElasticsearch](HdfsElasticsearch.md "HdfsElasticsearch" ) +[HdfsElasticsearch](src/site/markdown/HdfsElasticsearch.md "HdfsElasticsearch" ) Build: --------- http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/pom.xml ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/pom.xml b/local/elasticsearch-hdfs/pom.xml index 8819319..3d07371 100644 --- a/local/elasticsearch-hdfs/pom.xml +++ b/local/elasticsearch-hdfs/pom.xml @@ -29,7 +29,9 @@ <modelVersion>4.0.0</modelVersion> <artifactId>elasticsearch-hdfs</artifactId> - <version>0.2-incubating-SNAPSHOT</version> + <name>elasticsearch-hdfs</name> + + <description>Copies documents between elasticsearch and file system using the hdfs persist module.</description> <properties> <elasticsearch.version>1.1.0</elasticsearch.version> @@ -71,45 +73,38 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.apache.streams</groupId> - <artifactId>streams-core</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> <groupId>com.typesafe</groupId> <artifactId>config</artifactId> </dependency> <dependency> <groupId>org.apache.streams</groupId> + <artifactId>streams-core</artifactId> + </dependency> + <dependency> + <groupId>org.apache.streams</groupId> <artifactId>streams-config</artifactId> - <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-util</artifactId> - <version>${project.version}</version> <type>test-jar</type> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-pojo</artifactId> - <version>${project.version}</version> <type>test-jar</type> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-runtime-local</artifactId> - <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-persist-elasticsearch</artifactId> - <version>0.2-incubating-SNAPSHOT</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-persist-hdfs</artifactId> - <version>0.2-incubating-SNAPSHOT</version> <exclusions> <exclusion> <groupId>org.slf4j</groupId> http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/main/resources/ElasticsearchHdfs.dot.png ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/src/main/resources/ElasticsearchHdfs.dot.png b/local/elasticsearch-hdfs/src/main/resources/ElasticsearchHdfs.dot.png new file mode 100644 index 0000000..fd5f3b5 Binary files /dev/null and b/local/elasticsearch-hdfs/src/main/resources/ElasticsearchHdfs.dot.png differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/main/resources/HdfsElasticsearch.dot.png ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/src/main/resources/HdfsElasticsearch.dot.png b/local/elasticsearch-hdfs/src/main/resources/HdfsElasticsearch.dot.png new file mode 100644 index 0000000..d44f8db Binary files /dev/null and b/local/elasticsearch-hdfs/src/main/resources/HdfsElasticsearch.dot.png differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/markdown/ElasticsearchHdfs.md ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/src/site/markdown/ElasticsearchHdfs.md b/local/elasticsearch-hdfs/src/site/markdown/ElasticsearchHdfs.md new file mode 100644 index 0000000..219516a --- /dev/null +++ b/local/elasticsearch-hdfs/src/site/markdown/ElasticsearchHdfs.md @@ -0,0 +1,47 @@ +Apache Streams (incubating) +Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0 +-------------------------------------------------------------------------------- + +elasticsearch-hdfs +============================== + +Description: +----------------- + +Copies documents from elasticsearch to hdfs. + +Specification: +----------------- + +[ElasticsearchHdfs.dot](ElasticsearchHdfs.dot "ElasticsearchHdfs.dot" ) + +Diagram: +----------------- + +![ElasticsearchHdfs.dot.png](./ElasticsearchHdfs.dot.png?raw=true) + +Example Configuration: +---------------------- + + { + "source": { + "host": "localhost", + "port": 27017, + "db": "streams", + "collection": "activities" + }, + "destination": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "index": "destination", + "type": "activity" + } + } + +Run: +-------- + +`docker run elasticsearch-hdfs:0.2-incubating-SNAPSHOT-PP.jar java -cp stash-migrate-0.2-incubating-SNAPSHOT.jar -Dconfig.file=http://<location_of_config_file>.json org.apache.streams.elasticsearch.example.HdfsElasticsearch` http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/markdown/HdfsElasticsearch.md ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/src/site/markdown/HdfsElasticsearch.md b/local/elasticsearch-hdfs/src/site/markdown/HdfsElasticsearch.md new file mode 100644 index 0000000..db6b862 --- /dev/null +++ b/local/elasticsearch-hdfs/src/site/markdown/HdfsElasticsearch.md @@ -0,0 +1,50 @@ +Apache Streams (incubating) +Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0 +-------------------------------------------------------------------------------- + +hdfs-elasticsearch +============================== + +Description: +----------------- + +Copies documents from hdfs to elasticsearch. + +Specification: +----------------- + +[HdfsElasticsearch.dot](HdfsElasticsearch.dot "HdfsElasticsearch.dot" ) + +Diagram: +----------------- + +![HdfsElasticsearch.dot.png](./HdfsElasticsearch.dot.png?raw=true) + +Example Configuration: +---------------------- + + { + "source": { + "scheme": "file", + "host": "localhost", + "user": "cloudera", + "path": "/tmp", + "writerPath": "activity" + }, + "destination": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "index": "activity2", + "type": "activity", + "forceUseConfig": true + } + } + + +Run: +-------- + +`docker run elasticsearch-hdfs:0.2-incubating-SNAPSHOT.jar java -cp elasticsearch-hdfs-0.2-incubating-SNAPSHOT.jar -Dconfig.file=file://<location_of_config_file>.json org.apache.streams.elasticsearch.example.HdfsElasticsearch` http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/src/site/markdown/index.md b/local/elasticsearch-hdfs/src/site/markdown/index.md new file mode 100644 index 0000000..cfd5a30 --- /dev/null +++ b/local/elasticsearch-hdfs/src/site/markdown/index.md @@ -0,0 +1,35 @@ +Apache Streams (incubating) +Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0 +-------------------------------------------------------------------------------- + +elasticsearch-hdfs +============================== + +Requirements: +------------- + - A running ElasticSearch 1.0.0+ instance + +Description: +------------ +Copies documents between elasticsearch and file system using the hdfs persist module. + +Streams: +-------- + +<a href="HdfsElasticsearch.html" target="_self">HdfsElasticsearch</a> + +<a href="ElasticsearchHdfs.html" target="_self">ElasticsearchHdfs</a> + +Build: +--------- + +`mvn clean package` + +Note that an alternative version of hdfs is packaged, by excluding org.apache.hadoop.hadoop-hdfs when +importing org.apache.streams.streams-persist-hdfs, and specifically depending on a different preferred version. + +Deploy: +-------- + +`mvn -Pdocker clean package docker:build` + http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfs.dot ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfs.dot b/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfs.dot new file mode 100644 index 0000000..c953522 --- /dev/null +++ b/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfs.dot @@ -0,0 +1,17 @@ +digraph g { + + //providers + ElasticsearchPersistReader [label="ElasticsearchPersistReader",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java"]; + + //persisters + WebHdfsPersistWriter [label="WebHdfsPersistWriter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java"]; + + //data + source [label="es://{indexes}/{types}",shape=box]; + destination [label="hdfs://{index}/{type}",shape=box]; + + //stream + source -> ElasticsearchPersistReader + ElasticsearchPersistReader -> WebHdfsPersistWriter [label="String"]; + WebHdfsPersistWriter -> destination +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfs.dot.png ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfs.dot.png b/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfs.dot.png new file mode 100644 index 0000000..fd5f3b5 Binary files /dev/null and b/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfs.dot.png differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfsConfiguration.json ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfsConfiguration.json b/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfsConfiguration.json new file mode 100644 index 0000000..91324dd --- /dev/null +++ b/local/elasticsearch-hdfs/src/site/resources/ElasticsearchHdfsConfiguration.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema", + "type": "object", + "javaType" : "org.apache.streams.elasticsearch.example.ElasticsearchHdfsConfiguration", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "source": { "javaType": "org.apache.streams.elasticsearch.ElasticsearchReaderConfiguration", "type": "object", "required": true }, + "destination": { "javaType": "org.apache.streams.hdfs.HdfsWriterConfiguration", "type": "object", "required": true } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearch.dot ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearch.dot b/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearch.dot new file mode 100644 index 0000000..f9a0efa --- /dev/null +++ b/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearch.dot @@ -0,0 +1,17 @@ +digraph g { + + //providers + WebHdfsPersistReader [label="WebHdfsPersistReader",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java"]; + + //persisters + ElasticsearchPersistWriter [label="ElasticsearchPersistWriter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java"]; + + //data + source [label="hdfs://{indexes}/{types}",shape=box]; + destination [label="es://{index}/{type}",shape=box]; + + //stream + source -> WebHdfsPersistReader + WebHdfsPersistReader -> ElasticsearchPersistWriter [label="String"]; + ElasticsearchPersistWriter -> destination +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearch.dot.png ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearch.dot.png b/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearch.dot.png new file mode 100644 index 0000000..d44f8db Binary files /dev/null and b/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearch.dot.png differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearchConfiguration.json ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearchConfiguration.json b/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearchConfiguration.json new file mode 100644 index 0000000..c8072ba --- /dev/null +++ b/local/elasticsearch-hdfs/src/site/resources/HdfsElasticsearchConfiguration.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema", + "type": "object", + "javaType" : "org.apache.streams.elasticsearch.example.HdfsElasticsearchConfiguration", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "source": { "javaType": "org.apache.streams.hdfs.HdfsReaderConfiguration", "type": "object", "required": true }, + "destination": { "javaType": "org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration", "type": "object", "required": true } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/resources/testBackup.json ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/src/site/resources/testBackup.json b/local/elasticsearch-hdfs/src/site/resources/testBackup.json new file mode 100644 index 0000000..2002cb8 --- /dev/null +++ b/local/elasticsearch-hdfs/src/site/resources/testBackup.json @@ -0,0 +1,24 @@ +{ + "source": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "indexes": [ + "source" + ], + "types": [ + "activity" + ] + }, + "destination": { + "scheme": "file", + "host": "localhost", + "user": "cloudera", + "path": "target", + "writerPath": "test", + "writerFilePrefix": "activities" + } + +} http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-hdfs/src/site/resources/testRestore.json ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/src/site/resources/testRestore.json b/local/elasticsearch-hdfs/src/site/resources/testRestore.json new file mode 100644 index 0000000..63b42c5 --- /dev/null +++ b/local/elasticsearch-hdfs/src/site/resources/testRestore.json @@ -0,0 +1,20 @@ +{ + "source": { + "scheme": "file", + "host": "localhost", + "user": "cloudera", + "path": "target", + "readerPath": "test" + }, + "destination": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "index": "destination", + "type": "activity", + "forceUseConfig": true + } + +} http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/ElasticsearchReindex.png ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/ElasticsearchReindex.png b/local/elasticsearch-reindex/ElasticsearchReindex.png deleted file mode 100644 index 9e59093..0000000 Binary files a/local/elasticsearch-reindex/ElasticsearchReindex.png and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/README.md ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/README.md b/local/elasticsearch-reindex/README.md index 5fbace2..71e8c28 100644 --- a/local/elasticsearch-reindex/README.md +++ b/local/elasticsearch-reindex/README.md @@ -13,7 +13,7 @@ Requirements: Description: ------------ -Copies documents into a different index +Copies documents between indexes. Specification: ----------------- @@ -23,7 +23,7 @@ Specification: Diagram: ----------------- -![ElasticsearchReindex.png](./ElasticsearchReindex.png?raw=true) +![ElasticsearchReindex.dot.png](src/main/resources/ElasticsearchReindex.dot.png?raw=true) Example Configuration: ---------------------- http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/pom.xml ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/pom.xml b/local/elasticsearch-reindex/pom.xml index a75fa2a..59e968b 100644 --- a/local/elasticsearch-reindex/pom.xml +++ b/local/elasticsearch-reindex/pom.xml @@ -29,7 +29,9 @@ <modelVersion>4.0.0</modelVersion> <artifactId>elasticsearch-reindex</artifactId> - <version>0.2-incubating-SNAPSHOT</version> + <name>elasticsearch-reindex</name> + + <description>Copies documents between indexes.</description> <properties> <elasticsearch.version>1.1.0</elasticsearch.version> @@ -63,40 +65,34 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.apache.streams</groupId> - <artifactId>streams-core</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> <groupId>com.typesafe</groupId> <artifactId>config</artifactId> </dependency> <dependency> <groupId>org.apache.streams</groupId> + <artifactId>streams-core</artifactId> + </dependency> + <dependency> + <groupId>org.apache.streams</groupId> <artifactId>streams-config</artifactId> - <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-util</artifactId> - <version>${project.version}</version> <type>test-jar</type> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-pojo</artifactId> - <version>${project.version}</version> <type>test-jar</type> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-runtime-local</artifactId> - <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-persist-elasticsearch</artifactId> - <version>0.2-incubating-SNAPSHOT</version> </dependency> <dependency> <groupId>org.slf4j</groupId> http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/main/resources/ElasticsearchReindex.dot.png ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/src/main/resources/ElasticsearchReindex.dot.png b/local/elasticsearch-reindex/src/main/resources/ElasticsearchReindex.dot.png new file mode 100644 index 0000000..b6541bb Binary files /dev/null and b/local/elasticsearch-reindex/src/main/resources/ElasticsearchReindex.dot.png differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/src/site/markdown/index.md b/local/elasticsearch-reindex/src/site/markdown/index.md new file mode 100644 index 0000000..df20698 --- /dev/null +++ b/local/elasticsearch-reindex/src/site/markdown/index.md @@ -0,0 +1,79 @@ +Apache Streams (incubating) +Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0 +-------------------------------------------------------------------------------- + +elasticsearch-reindex +============================== + +Requirements: +------------- + - A running ElasticSearch 1.0.0+ cluster + - Transport client access to cluster + - elasticsearch.version and lucene.version set to match cluster + +Description: +------------ +Copies documents into a different index + +Specification: +----------------- + +[ElasticsearchReindex.dot](src/main/resources/ElasticsearchReindex.dot "ElasticsearchReindex.dot" ) + +Diagram: +----------------- + +![ElasticsearchReindex.dot.png](./ElasticsearchReindex.dot.png?raw=true) + +Example Configuration: +---------------------- + + { + "source": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "indexes": [ + "activity" + ], + "types": [ + "activity" + ], + "forceUseConfig": true + }, + "destination": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "index": "activity2", + "type": "activity", + "forceUseConfig": true + } + } + +Populate source and destination in configuration with cluster / index / type details. + +Build: +--------- + +`mvn clean package verify` + +Run: +-------- + +`java -cp target/elasticsearch-reindex-0.1-SNAPSHOT.jar -Dconfig.file=src/main/resources/application.json org.apache.streams.example.elasticsearch.ElasticsearchReindex` + +Deploy: +-------- +`mvn -Pdocker clean package docker:build` + +`docker tag elasticsearch-reindex:0.2-incubating-SNAPSHOT <dockerregistry>:elasticsearch-reindex:0.2-incubating-SNAPSHOT` + +`docker push <dockerregistry>:elasticsearch-reindex:0.2-incubating-SNAPSHOT` + +`docker run <dockerregistry>:elasticsearch-reindex:0.2-incubating-SNAPSHOT.jar java -cp elasticsearch-reindex-0.2-incubating-SNAPSHOT.jar -Dconfig.file=http://<location_of_config_file>.json org.apache.streams.example.elasticsearch.ElasticsearchReindex` + http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/resources/ActivityChildObjectParent.json ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/src/site/resources/ActivityChildObjectParent.json b/local/elasticsearch-reindex/src/site/resources/ActivityChildObjectParent.json new file mode 100644 index 0000000..963fbe7 --- /dev/null +++ b/local/elasticsearch-reindex/src/site/resources/ActivityChildObjectParent.json @@ -0,0 +1,15 @@ +{ + "template": "*", + "order": 100, + "mappings": { + "object": { + "dynamic": true + }, + "activity": { + "_parent": { + "type": "object" + }, + "dynamic": true + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindex.dot ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindex.dot b/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindex.dot new file mode 100644 index 0000000..52456a9 --- /dev/null +++ b/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindex.dot @@ -0,0 +1,17 @@ +digraph g { + + //providers + ElasticsearchPersistReader [label="ElasticsearchPersistReader",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java"]; + + //persisters + ElasticsearchPersistWriter [label="ElasticsearchPersistWriter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java"]; + + //data + source [label="es://{indexes}/{types}",shape=box]; + destination [label="es://{index}/{type}",shape=box]; + + //stream + source -> ElasticsearchPersistReader + ElasticsearchPersistReader -> ElasticsearchPersistWriter [label="String"]; + ElasticsearchPersistWriter -> destination +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindex.dot.png ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindex.dot.png b/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindex.dot.png new file mode 100644 index 0000000..b6541bb Binary files /dev/null and b/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindex.dot.png differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindexConfiguration.json ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindexConfiguration.json b/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindexConfiguration.json new file mode 100644 index 0000000..ffdc4d8 --- /dev/null +++ b/local/elasticsearch-reindex/src/site/resources/ElasticsearchReindexConfiguration.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema", + "type": "object", + "javaType" : "org.apache.streams.elasticsearch.example.ElasticsearchReindexConfiguration", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "source": { "javaType": "org.apache.streams.elasticsearch.ElasticsearchReaderConfiguration", "type": "object", "required": true }, + "destination": { "javaType": "org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration", "type": "object", "required": true } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/resources/application.json ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/src/site/resources/application.json b/local/elasticsearch-reindex/src/site/resources/application.json new file mode 100644 index 0000000..567b19d --- /dev/null +++ b/local/elasticsearch-reindex/src/site/resources/application.json @@ -0,0 +1,26 @@ +{ + "source": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "indexes": [ + "activity" + ], + "types": [ + "activity" + ], + "forceUseConfig": true + }, + "destination": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "index": "activity2", + "type": "activity", + "forceUseConfig": true + } +} http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/resources/testReindex.json ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/src/site/resources/testReindex.json b/local/elasticsearch-reindex/src/site/resources/testReindex.json new file mode 100644 index 0000000..0cc5aa0 --- /dev/null +++ b/local/elasticsearch-reindex/src/site/resources/testReindex.json @@ -0,0 +1,26 @@ +{ + "source": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "indexes": [ + "source" + ], + "types": [ + "activity" + ] + }, + "destination": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "index": "destination", + "type": "activity", + "forceUseConfig": true + } + +} http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/resources/testReindexChild.json ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/src/site/resources/testReindexChild.json b/local/elasticsearch-reindex/src/site/resources/testReindexChild.json new file mode 100644 index 0000000..0cc5aa0 --- /dev/null +++ b/local/elasticsearch-reindex/src/site/resources/testReindexChild.json @@ -0,0 +1,26 @@ +{ + "source": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "indexes": [ + "source" + ], + "types": [ + "activity" + ] + }, + "destination": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "index": "destination", + "type": "activity", + "forceUseConfig": true + } + +} http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/elasticsearch-reindex/src/site/resources/testReindexParent.json ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/src/site/resources/testReindexParent.json b/local/elasticsearch-reindex/src/site/resources/testReindexParent.json new file mode 100644 index 0000000..c601cea --- /dev/null +++ b/local/elasticsearch-reindex/src/site/resources/testReindexParent.json @@ -0,0 +1,26 @@ +{ + "source": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "indexes": [ + "source" + ], + "types": [ + "object" + ] + }, + "destination": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "index": "destination", + "type": "object", + "forceUseConfig": true + } + +} http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/MongoElasticsearchSync.png ---------------------------------------------------------------------- diff --git a/local/mongo-elasticsearch-sync/MongoElasticsearchSync.png b/local/mongo-elasticsearch-sync/MongoElasticsearchSync.png deleted file mode 100644 index a8fc4d7..0000000 Binary files a/local/mongo-elasticsearch-sync/MongoElasticsearchSync.png and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/README.md ---------------------------------------------------------------------- diff --git a/local/mongo-elasticsearch-sync/README.md b/local/mongo-elasticsearch-sync/README.md index 72b11db..ddfb84b 100644 --- a/local/mongo-elasticsearch-sync/README.md +++ b/local/mongo-elasticsearch-sync/README.md @@ -12,7 +12,7 @@ Requirements: Description: ------------ -Copies documents from mongodb to elasticsearch +Copies documents from mongodb to elasticsearch. Specification: ----------------- @@ -22,7 +22,7 @@ Specification: Diagram: ----------------- -![MongoElasticsearchSync.png](./MongoElasticsearchSync.png?raw=true) +![MongoElasticsearchSync.dot.png](src/main/resources/MongoElasticsearchSync.dot.png?raw=true) Example Configuration: ---------------------- http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/pom.xml ---------------------------------------------------------------------- diff --git a/local/mongo-elasticsearch-sync/pom.xml b/local/mongo-elasticsearch-sync/pom.xml index 4d271e0..411bd9b 100644 --- a/local/mongo-elasticsearch-sync/pom.xml +++ b/local/mongo-elasticsearch-sync/pom.xml @@ -29,6 +29,9 @@ <modelVersion>4.0.0</modelVersion> <artifactId>mongo-elasticsearch-sync</artifactId> + <name>mongo-elasticsearch-sync</name> + + <description>Copies documents from mongodb to elasticsearch.</description> <properties> <elasticsearch.version>1.1.0</elasticsearch.version> http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/src/main/resources/MongoElasticsearchSync.dot.png ---------------------------------------------------------------------- diff --git a/local/mongo-elasticsearch-sync/src/main/resources/MongoElasticsearchSync.dot.png b/local/mongo-elasticsearch-sync/src/main/resources/MongoElasticsearchSync.dot.png new file mode 100644 index 0000000..b898bfc Binary files /dev/null and b/local/mongo-elasticsearch-sync/src/main/resources/MongoElasticsearchSync.dot.png differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/local/mongo-elasticsearch-sync/src/site/markdown/index.md b/local/mongo-elasticsearch-sync/src/site/markdown/index.md new file mode 100644 index 0000000..c4f0b9e --- /dev/null +++ b/local/mongo-elasticsearch-sync/src/site/markdown/index.md @@ -0,0 +1,68 @@ +Apache Streams (incubating) +Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0 +-------------------------------------------------------------------------------- + +mongo-elasticsearch-sync +============================== + +Requirements: +------------- + - A running MongoDB 2.4+ instance + - A running ElasticSearch 1.0.0+ instance + +Description: +------------ +Copies documents from mongodb to elasticsearch + +Specification: +----------------- + +[MongoElasticsearchSync.dot](MongoElasticsearchSync.dot "MongoElasticsearchSync.dot" ) + +Diagram: +----------------- + +![MongoElasticsearchSync.dot.png](./MongoElasticsearchSync.dot.png?raw=true) + +Example Configuration: +---------------------- + + { + "source": { + "host": "localhost", + "port": 27017, + "db": "streams", + "collection": "activities" + }, + "destination": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "index": "destination", + "type": "activity" + } + } + +Build: +--------- + +`mvn clean package verify` + +Run: +-------- + +`java -cp target/mongo-elasticsearch-sync-0.1-SNAPSHOT.jar -Dconfig.file=src/main/resources/application.json org.apache.streams.example.elasticsearch.MongoElasticsearchSync` + +Deploy: +-------- + +`mvn -Pdocker clean package docker:build` + +`docker tag mongo-elasticsearch-sync:0.2-incubating-SNAPSHOT <dockerregistry>:mongo-elasticsearch-sync:0.2-incubating-SNAPSHOT` + +`docker push <dockerregistry>:mongo-elasticsearch-sync:0.2-incubating-SNAPSHOT` + +`docker run <dockerregistry>:mongo-elasticsearch-sync:0.2-incubating-SNAPSHOT java -cp mongo-elasticsearch-sync-0.2-incubating-SNAPSHOT.jar -Dconfig.file=http://<location_of_config_file>.json org.apache.streams.example.elasticsearch.MongoElasticsearchSync` + http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSync.dot ---------------------------------------------------------------------- diff --git a/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSync.dot b/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSync.dot new file mode 100644 index 0000000..e3babf5 --- /dev/null +++ b/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSync.dot @@ -0,0 +1,18 @@ +digraph g { + + //providers + MongoPersistReader [label="MongoPersistReader",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-mongo/src/main/java/org/apache/streams/mongo/MongoPersistReader.java"]; + + //persisters + ElasticsearchPersistWriter [label="ElasticsearchPersistWriter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java"]; + + //data + source [label="mongdb://{db}/{collection}",shape=box]; + destination [label="es://{index}/{type}",shape=box]; + + //stream + source -> MongoPersistReader [label="ObjectNode"]; + MongoPersistReader -> ElasticsearchPersistWriter [label="ObjectNode"]; + ElasticsearchPersistWriter -> destination [label="ObjectNode"]; + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSync.dot.png ---------------------------------------------------------------------- diff --git a/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSync.dot.png b/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSync.dot.png new file mode 100644 index 0000000..b898bfc Binary files /dev/null and b/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSync.dot.png differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSyncConfiguration.json ---------------------------------------------------------------------- diff --git a/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSyncConfiguration.json b/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSyncConfiguration.json new file mode 100644 index 0000000..a592699 --- /dev/null +++ b/local/mongo-elasticsearch-sync/src/site/resources/MongoElasticsearchSyncConfiguration.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema", + "type": "object", + "javaType" : "org.apache.streams.example.elasticsearch.MongoElasticsearchSyncConfiguration", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "source": { "javaType": "org.apache.streams.mongo.MongoConfiguration", "type": "object", "required": true }, + "destination": { "javaType": "org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration", "type": "object", "required": true } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/src/site/resources/application.json ---------------------------------------------------------------------- diff --git a/local/mongo-elasticsearch-sync/src/site/resources/application.json b/local/mongo-elasticsearch-sync/src/site/resources/application.json new file mode 100644 index 0000000..facb336 --- /dev/null +++ b/local/mongo-elasticsearch-sync/src/site/resources/application.json @@ -0,0 +1,17 @@ +{ + "source": { + "host": "localhost", + "port": 27017, + "db": "streams", + "collection": "activities" + }, + "destination": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "index": "destination", + "type": "activity" + } +} http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/mongo-elasticsearch-sync/src/site/resources/testSync.json ---------------------------------------------------------------------- diff --git a/local/mongo-elasticsearch-sync/src/site/resources/testSync.json b/local/mongo-elasticsearch-sync/src/site/resources/testSync.json new file mode 100644 index 0000000..b678c21 --- /dev/null +++ b/local/mongo-elasticsearch-sync/src/site/resources/testSync.json @@ -0,0 +1,18 @@ +{ + "source": { + "host": "localhost", + "port": 37017, + "db": "local", + "collection": "activities" + }, + "destination": { + "hosts": [ + "localhost" + ], + "port": 9300, + "clusterName": "elasticsearch", + "index": "destination", + "type": "activity", + "forceUseConfig": true + } +} http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-follow-graph/README.md ---------------------------------------------------------------------- diff --git a/local/twitter-follow-graph/README.md b/local/twitter-follow-graph/README.md index 5921df8..e7e3bce 100644 --- a/local/twitter-follow-graph/README.md +++ b/local/twitter-follow-graph/README.md @@ -22,7 +22,7 @@ Specification: Diagram: ----------------- -![TwitterFollowGraph.png](./TwitterFollowGraph.png?raw=true) +![TwitterFollowGraph.dot.png](src/main/resources/TwitterFollowGraph.dot.png?raw=true) Example Configuration: ---------------------- http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-follow-graph/TwitterFollowGraph.png ---------------------------------------------------------------------- diff --git a/local/twitter-follow-graph/TwitterFollowGraph.png b/local/twitter-follow-graph/TwitterFollowGraph.png deleted file mode 100644 index b84db6c..0000000 Binary files a/local/twitter-follow-graph/TwitterFollowGraph.png and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-follow-graph/pom.xml ---------------------------------------------------------------------- diff --git a/local/twitter-follow-graph/pom.xml b/local/twitter-follow-graph/pom.xml index 5324462..db827e2 100644 --- a/local/twitter-follow-graph/pom.xml +++ b/local/twitter-follow-graph/pom.xml @@ -23,37 +23,38 @@ <parent> <groupId>org.apache.streams</groupId> <artifactId>streams-examples-local</artifactId> - <version>0.2-incubating-SNAPSHOT</version> + <version>0.3-incubating-SNAPSHOT</version> <relativePath>..</relativePath> </parent> <modelVersion>4.0.0</modelVersion> <artifactId>twitter-follow-graph</artifactId> + <name>twitter-follow-graph</name> + + <description> + Collects friend or follower connections for a set of twitter users to build a graph database in neo4j. + </description> <dependencies> <dependency> - <groupId>org.apache.streams</groupId> - <artifactId>streams-core</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> <groupId>com.typesafe</groupId> <artifactId>config</artifactId> </dependency> <dependency> <groupId>org.apache.streams</groupId> + <artifactId>streams-core</artifactId> + </dependency> + <dependency> + <groupId>org.apache.streams</groupId> <artifactId>streams-config</artifactId> - <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-runtime-local</artifactId> - <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-provider-twitter</artifactId> - <version>${project.version}</version> <exclusions> <exclusion> <groupId>commons-logging</groupId> @@ -64,12 +65,10 @@ <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-persist-graph</artifactId> - <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-pojo</artifactId> - <version>${project.version}</version> <type>test-jar</type> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-follow-graph/src/main/resources/TwitterFollowGraph.dot.png ---------------------------------------------------------------------- diff --git a/local/twitter-follow-graph/src/main/resources/TwitterFollowGraph.dot.png b/local/twitter-follow-graph/src/main/resources/TwitterFollowGraph.dot.png new file mode 100644 index 0000000..10dc842 Binary files /dev/null and b/local/twitter-follow-graph/src/main/resources/TwitterFollowGraph.dot.png differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-follow-graph/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/local/twitter-follow-graph/src/site/markdown/index.md b/local/twitter-follow-graph/src/site/markdown/index.md new file mode 100644 index 0000000..a813f19 --- /dev/null +++ b/local/twitter-follow-graph/src/site/markdown/index.md @@ -0,0 +1,97 @@ +Apache Streams (incubating) +Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0 +-------------------------------------------------------------------------------- + +twitter-follow-graph +============================== + +Requirements: +------------- + - Authorized Twitter API credentials + - A running Neo4J 1.9.0+ instance + +Description: +------------ +Collects friend or follower connections for a set of twitter users to build a graph database in neo4j. + +Specification: +----------------- + +[TwitterFollowGraph.dot](TwitterFollowGraph.dot "TwitterFollowGraph.dot" ) + +Diagram: +----------------- + +![TwitterFollowGraph.dot.png](./TwitterFollowGraph.dot.png?raw=true) + +Example Configuration: +---------------------- + + { + "twitter": { + "endpoint": "friends", + "oauth": { + "consumerSecret": "", + "consumerKey": "", + "accessToken": "", + "accessTokenSecret": "" + }, + "info": [ + 42232950 + ] + }, + "graph": { + "vertices": { + "objects": [ + "actor", + "object" + ], + "verbs": [ + "follow" + ], + "objectTypes": [ + "page" + ] + }, + "edges": { + "objects": [ + "actor", + "object" + ], + "verbs": [ + "follow" + ], + "objectTypes": [ + "page" + ] + }, + "protocol": "http", + "hostname": "localhost", + "port": 7474, + "requestMethod": "GET", + "content-type": "application/json", + "type": "neo4j", + "graph": "data" + } + } + +Build: +--------- + +`mvn clean package verify` + +Run: +-------- + +`java -cp target/twitter-follow-graph-0.2-incubating-SNAPSHOT.jar -Dconfig.file=src/main/resources/application.json org.apache.streams.example.graph.TwitterFollowGraph` + +Deploy: +-------- +`mvn -Pdocker clean package docker:build` + +`docker tag twitter-follow-graph:0.2-incubating-SNAPSHOT <dockerregistry>:twitter-follow-graph:0.2-incubating-SNAPSHOT` + +`docker push <dockerregistry>:twitter-follow-graph:0.2-incubating-SNAPSHOT` + +`docker run <dockerregistry>:twitter-follow-graph:0.2-incubating-SNAPSHOT java -cp twitter-follow-graph-0.2-incubating-SNAPSHOT.jar -Dconfig.file=http://<location_of_config_file>.json org.apache.streams.example.graph.TwitterFollowGraph` + http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-follow-graph/src/site/resources/TwitterFollowGraph.dot ---------------------------------------------------------------------- diff --git a/local/twitter-follow-graph/src/site/resources/TwitterFollowGraph.dot b/local/twitter-follow-graph/src/site/resources/TwitterFollowGraph.dot new file mode 100644 index 0000000..c14a2ab --- /dev/null +++ b/local/twitter-follow-graph/src/site/resources/TwitterFollowGraph.dot @@ -0,0 +1,21 @@ +digraph g { + + //providers + TwitterFollowingProvider [label="TwitterFollowingProvider",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterFollowingProvider.java"]; + + //processors + TypeConverterProcessor [label="TypeConverterProcessor",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-components/streams-converters/src/main/java/org/apache/streams/converters/TypeConverterProcessor.java"]; + ActivityConverterProcessor [label="ActivityConverterProcessor",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-components/streams-converters/src/main/java/org/apache/streams/converters/ActivityConverterProcessor.java"]; + + //persisters + GraphPersistWriter [label="GraphPersistWriter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-graph/src/main/java/org/apache/streams/graph/GraphPersistWriter.java"]; + + //data + destination [label="http://{host}:{port}/db/data",shape=box]; + + //stream + TwitterFollowingProvider -> TypeConverterProcessor [label="Follow",URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Follow.java"]; + TypeConverterProcessor -> ActivityConverterProcessor [label="String"]; + ActivityConverterProcessor -> GraphPersistWriter [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"]; + GraphPersistWriter -> destination +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-follow-graph/src/site/resources/TwitterFollowGraph.dot.png ---------------------------------------------------------------------- diff --git a/local/twitter-follow-graph/src/site/resources/TwitterFollowGraph.dot.png b/local/twitter-follow-graph/src/site/resources/TwitterFollowGraph.dot.png new file mode 100644 index 0000000..10dc842 Binary files /dev/null and b/local/twitter-follow-graph/src/site/resources/TwitterFollowGraph.dot.png differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-follow-graph/src/site/resources/TwitterFollowGraphConfiguration.json ---------------------------------------------------------------------- diff --git a/local/twitter-follow-graph/src/site/resources/TwitterFollowGraphConfiguration.json b/local/twitter-follow-graph/src/site/resources/TwitterFollowGraphConfiguration.json new file mode 100644 index 0000000..c2ccc80 --- /dev/null +++ b/local/twitter-follow-graph/src/site/resources/TwitterFollowGraphConfiguration.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema", + "$license": [ + "http://www.apache.org/licenses/LICENSE-2.0" + ], + "type": "object", + "javaType" : "org.apache.streams.example.graph.TwitterFollowingGraphConfiguration", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "twitter": { "javaType": "org.apache.streams.twitter.TwitterUserInformationConfiguration", "type": "object", "required": true }, + "graph": { "javaType": "org.apache.streams.graph.GraphWriterConfiguration", "type": "object", "required": true } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/README.md ---------------------------------------------------------------------- diff --git a/local/twitter-history-elasticsearch/README.md b/local/twitter-history-elasticsearch/README.md index 0ccce34..7e64a84 100644 --- a/local/twitter-history-elasticsearch/README.md +++ b/local/twitter-history-elasticsearch/README.md @@ -24,7 +24,7 @@ Specification: Diagram: ----------------- -![TwitterHistoryElasticsearch.png](./TwitterHistoryElasticsearch.png?raw=true) +![TwitterHistoryElasticsearch.dot.png](src/main/resources/TwitterHistoryElasticsearch.dot.png?raw=true) Example Configuration: ---------------------- http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png ---------------------------------------------------------------------- diff --git a/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png b/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png deleted file mode 100644 index 0a87125..0000000 Binary files a/local/twitter-history-elasticsearch/TwitterHistoryElasticsearch.png and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/pom.xml ---------------------------------------------------------------------- diff --git a/local/twitter-history-elasticsearch/pom.xml b/local/twitter-history-elasticsearch/pom.xml index 4adabb2..19b9e1c 100644 --- a/local/twitter-history-elasticsearch/pom.xml +++ b/local/twitter-history-elasticsearch/pom.xml @@ -28,6 +28,13 @@ <modelVersion>4.0.0</modelVersion> <artifactId>twitter-history-elasticsearch</artifactId> + <name>twitter-history-elasticsearch</name> + + <description> + Retrieves as many posts from a known list of users as twitter API allows. + + Converts them to activities, and writes them in activity format to Elasticsearch. + </description> <properties> <elasticsearch.version>1.1.0</elasticsearch.version> @@ -68,34 +75,28 @@ <dependency> <groupId>com.typesafe</groupId> <artifactId>config</artifactId> - <version>1.2.0</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-config</artifactId> - <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-util</artifactId> - <version>${project.version}</version> <type>test-jar</type> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-pojo</artifactId> - <version>${project.version}</version> <type>test-jar</type> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-runtime-local</artifactId> - <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-provider-twitter</artifactId> - <version>${project.version}</version> <exclusions> <exclusion> <groupId>commons-logging</groupId> @@ -106,7 +107,6 @@ <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-persist-elasticsearch</artifactId> - <version>${project.version}</version> </dependency> <dependency> <groupId>org.slf4j</groupId> http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot.png ---------------------------------------------------------------------- diff --git a/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot.png b/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot.png new file mode 100644 index 0000000..9be8aaf Binary files /dev/null and b/local/twitter-history-elasticsearch/src/main/resources/TwitterHistoryElasticsearch.dot.png differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/local/twitter-history-elasticsearch/src/site/markdown/index.md b/local/twitter-history-elasticsearch/src/site/markdown/index.md new file mode 100644 index 0000000..4afcf4e --- /dev/null +++ b/local/twitter-history-elasticsearch/src/site/markdown/index.md @@ -0,0 +1,78 @@ +Apache Streams (incubating) +Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0 +-------------------------------------------------------------------------------- + +twitter-history-elasticsearch +============================== + +Requirements: +------------- + - Authorized Twitter API credentials + - A running ElasticSearch 1.0.0+ instance + +Description: +------------ +Retrieves as many posts from a known list of users as twitter API allows. + +Converts them to activities, and writes them in activity format to Elasticsearch. + +Specification: +----------------- + +[TwitterHistoryElasticsearch.dot](src/main/resources/TwitterHistoryElasticsearch.dot "TwitterHistoryElasticsearch.dot" ) + +Diagram: +----------------- + +![TwitterHistoryElasticsearch.dot.png](./TwitterHistoryElasticsearch.dot.png?raw=true) + +Example Configuration: +---------------------- + + twitter { + â â host = "api.twitter.com" + â â endpoint = "statuses/user_timeline" + â â oauth { + â â â â consumerKey = "" + â â â â consumerSecret = "" + â â â â accessToken = "" + â â â â accessTokenSecret = "" + â â } + â â info = [ + "42232950" + "211620426" + ] + } + elasticsearch { + â â hosts = [ + â â â â localhost + â â ] + â â port = 9300 + â â clusterName = elasticsearch + â â index = userhistory_activity + â â type = activity + } + +In the Twitter section you should place all of your relevant authentication keys and whichever Twitter IDs you want to pull history for. + +Twitter IDs can be converted from screennames at http://www.gettwitterid.com + +Build: +--------- + +`mvn clean package verify` + +Run: +-------- + + java -cp target/twitter-history-elasticsearch-0.2-incubating-SNAPSHOT.jar -Dconfig.file=application.conf org.apache.streams.example.twitter.TwitterHistoryElasticsearch + +Deploy: +-------- +`mvn -Pdocker clean package docker:build` + +`docker tag twitter-history-elasticsearch:0.2-incubating-SNAPSHOT <dockerregistry>:elasticsearch-reindex:0.2-incubating-SNAPSHOT` + +`docker push <dockerregistry>:twitter-history-elasticsearch:0.2-incubating-SNAPSHOT` + +`docker run <dockerregistry>:twitter-history-elasticsearch:0.2-incubating-SNAPSHOT.jar java -cp twitter-history-elasticsearch-0.2-incubating-SNAPSHOT.jar -Dconfig.file=http://<location_of_config_file>.json org.apache.streams.example.twitter.TwitterHistoryElasticsearch` http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/src/site/resources/TwitterHistoryElasticsearch.dot ---------------------------------------------------------------------- diff --git a/local/twitter-history-elasticsearch/src/site/resources/TwitterHistoryElasticsearch.dot b/local/twitter-history-elasticsearch/src/site/resources/TwitterHistoryElasticsearch.dot new file mode 100644 index 0000000..525c61d --- /dev/null +++ b/local/twitter-history-elasticsearch/src/site/resources/TwitterHistoryElasticsearch.dot @@ -0,0 +1,21 @@ +digraph g { + + //providers + TwitterTimelineProvider [label="TwitterTimelineProvider",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java"]; + + //processors + ActivityConverterProcessor [label="ActivityConverterProcessor",shape=box,URL="https://github.com/apache/incubator-streams/blob/master/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityConverterProcessor.java"]; + + //persisters + ElasticsearchPersistWriter [label="ElasticsearchPersistWriter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java"]; + + //data + es [label="es://{index}/{type}",shape=box]; + + //stream + TwitterTimelineProvider -> ActivityConverterProcessor [label="ObjectNode"]; + ActivityConverterProcessor -> ElasticsearchPersistWriter [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"]; + + ElasticsearchPersistWriter -> es [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"]; + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/src/site/resources/TwitterHistoryElasticsearch.dot.png ---------------------------------------------------------------------- diff --git a/local/twitter-history-elasticsearch/src/site/resources/TwitterHistoryElasticsearch.dot.png b/local/twitter-history-elasticsearch/src/site/resources/TwitterHistoryElasticsearch.dot.png new file mode 100644 index 0000000..9be8aaf Binary files /dev/null and b/local/twitter-history-elasticsearch/src/site/resources/TwitterHistoryElasticsearch.dot.png differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/src/site/resources/application.conf ---------------------------------------------------------------------- diff --git a/local/twitter-history-elasticsearch/src/site/resources/application.conf b/local/twitter-history-elasticsearch/src/site/resources/application.conf new file mode 100644 index 0000000..c74b54b --- /dev/null +++ b/local/twitter-history-elasticsearch/src/site/resources/application.conf @@ -0,0 +1,23 @@ +twitter { + host = "api.twitter.com" + endpoint = "statuses/user_timeline" + oauth { + consumerKey = "" + consumerSecret = "" + accessToken = "" + accessTokenSecret = "" + } + info = [ + "42232950" + "211620426" + ] +} +elasticsearch { + hosts = [ + localhost + ] + port = 9300 + clusterName = elasticsearch + index = twitterhistory_activity + type = activity +} http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-history-elasticsearch/src/site/resources/org/apache/streams/example/twitter/TwitterHistoryElasticsearchConfiguration.json ---------------------------------------------------------------------- diff --git a/local/twitter-history-elasticsearch/src/site/resources/org/apache/streams/example/twitter/TwitterHistoryElasticsearchConfiguration.json b/local/twitter-history-elasticsearch/src/site/resources/org/apache/streams/example/twitter/TwitterHistoryElasticsearchConfiguration.json new file mode 100644 index 0000000..ea9b165 --- /dev/null +++ b/local/twitter-history-elasticsearch/src/site/resources/org/apache/streams/example/twitter/TwitterHistoryElasticsearchConfiguration.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema", + "$license": [ + "http://www.apache.org/licenses/LICENSE-2.0" + ], + "type": "object", + "javaType" : "org.apache.streams.example.twitter.TwitterHistoryElasticsearchConfiguration", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "twitter": { "javaType": "org.apache.streams.twitter.TwitterUserInformationConfiguration", "type": "object", "required": true }, + "elasticsearch": { "javaType": "org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration", "type": "object", "required": true } + } +} http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-userstream-elasticsearch/README.md ---------------------------------------------------------------------- diff --git a/local/twitter-userstream-elasticsearch/README.md b/local/twitter-userstream-elasticsearch/README.md index e2b4a2d..1b68afa 100644 --- a/local/twitter-userstream-elasticsearch/README.md +++ b/local/twitter-userstream-elasticsearch/README.md @@ -12,7 +12,7 @@ Requirements: Description: ------------ -This example connects to an active twitter account and stores the userstream as activities in Elasticsearch +Connects to an active twitter account and stores the userstream as activities in Elasticsearch Specification: ----------------- @@ -22,7 +22,7 @@ Specification: Diagram: ----------------- -![TwitterUserstreamElasticsearch.png](./TwitterUserstreamElasticsearch.png?raw=true) +![TwitterUserstreamElasticsearch.dot.png](./TwitterUserstreamElasticsearch.dot.png?raw=true) Example Configuration: ---------------------- http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-userstream-elasticsearch/TwitterUserstreamElasticsearch.png ---------------------------------------------------------------------- diff --git a/local/twitter-userstream-elasticsearch/TwitterUserstreamElasticsearch.png b/local/twitter-userstream-elasticsearch/TwitterUserstreamElasticsearch.png deleted file mode 100644 index 003a002..0000000 Binary files a/local/twitter-userstream-elasticsearch/TwitterUserstreamElasticsearch.png and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-userstream-elasticsearch/pom.xml ---------------------------------------------------------------------- diff --git a/local/twitter-userstream-elasticsearch/pom.xml b/local/twitter-userstream-elasticsearch/pom.xml index f438e77..6bb57b6 100644 --- a/local/twitter-userstream-elasticsearch/pom.xml +++ b/local/twitter-userstream-elasticsearch/pom.xml @@ -28,6 +28,11 @@ <modelVersion>4.0.0</modelVersion> <artifactId>twitter-userstream-elasticsearch</artifactId> + <name>twitter-userstream-elasticsearch</name> + + <description> + Connects to an active twitter account and stores the userstream as activities in Elasticsearch + </description> <properties> <elasticsearch.version>1.1.0</elasticsearch.version> @@ -63,7 +68,6 @@ <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-core</artifactId> - <version>${project.version}</version> </dependency> <dependency> <groupId>com.typesafe</groupId> @@ -72,32 +76,26 @@ <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-config</artifactId> - <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-runtime-local</artifactId> - <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-filters</artifactId> - <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-provider-twitter</artifactId> - <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-persist-elasticsearch</artifactId> - <version>${project.version}</version> </dependency> <dependency> <groupId>org.apache.streams</groupId> <artifactId>streams-pojo</artifactId> - <version>${project.version}</version> <type>test-jar</type> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-userstream-elasticsearch/src/main/resources/TwitterUserstreamElasticsearch.dot.png ---------------------------------------------------------------------- diff --git a/local/twitter-userstream-elasticsearch/src/main/resources/TwitterUserstreamElasticsearch.dot.png b/local/twitter-userstream-elasticsearch/src/main/resources/TwitterUserstreamElasticsearch.dot.png new file mode 100644 index 0000000..6e8713f Binary files /dev/null and b/local/twitter-userstream-elasticsearch/src/main/resources/TwitterUserstreamElasticsearch.dot.png differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-userstream-elasticsearch/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/local/twitter-userstream-elasticsearch/src/site/markdown/index.md b/local/twitter-userstream-elasticsearch/src/site/markdown/index.md new file mode 100644 index 0000000..1ecf179 --- /dev/null +++ b/local/twitter-userstream-elasticsearch/src/site/markdown/index.md @@ -0,0 +1,73 @@ +Apache Streams (incubating) +Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0 +-------------------------------------------------------------------------------- + +twitter-userstream-elasticsearch +============================== + +Requirements: +------------- + - Authorized Twitter API credentials + - A running ElasticSearch 1.0.0+ instance + +Description: +------------ +This example connects to an active twitter account and stores the userstream as activities in Elasticsearch + +Specification: +----------------- + +[TwitterUserstreamElasticsearch.dot](TwitterUserstreamElasticsearch.dot "TwitterUserstreamElasticsearch.dot" ) + +Diagram: +----------------- + +![TwitterUserstreamElasticsearch.dot.png](./TwitterUserstreamElasticsearch.dot.png?raw=true) + +Example Configuration: +---------------------- + + twitter { + â â endpoint = "userstream" + â â oauth { + â â â â â â consumerKey = "bcg14JThZEGoZ3MZOoT2HnJS7" + â â â â â â consumerSecret = "S4dwxnZni58CIJaoupGnUrO4HRHmbBGOb28W6IqOJBx36LPw2z" + â â â â â â accessToken = "" + â â â â â â accessTokenSecret = "" + } + } + elasticsearch { + â â hosts = [ + â â â â localhost + â â ] + â â port = 9300 + â â clusterName = elasticsearch + â â index = userstream_activity + â â type = activity + batchSize = 1 + } + +The consumerKey and consumerSecret are set for our streams-example application +The accessToken and accessTokenSecret can be obtained by navigating to: + + https://api.twitter.com/oauth/authenticate?oauth_token=UIJ0AUxCJatpKDUyFt0OTSEP4asZgqxRwUCT0AMSwc&oauth_callback=http%3A%2F%2Foauth.streamstutorial.w2odata.com%3A8080%2Fsocialauthdemo%2FsocialAuthSuccessAction.do + +Build: +--------- + +`mvn clean package verify` + +Run: +-------- + +`java -cp target/twitter-userstream-elasticsearch-0.2-incubating-SNAPSHOT.jar -Dconfig.file=src/main/resources/application.json org.apache.streams.example.twitter.TwitterUserstreamElasticsearch` + +Deploy: +-------- +`mvn -Pdocker clean package docker:build` + +`docker tag twitter-userstream-elasticsearch:0.2-incubating-SNAPSHOT <dockerregistry>:twitter-userstream-elasticsearch:0.2-incubating-SNAPSHOT` + +`docker push <dockerregistry>:twitter-userstream-elasticsearch:0.2-incubating-SNAPSHOT` + +`docker run <dockerregistry>:twitter-userstream-elasticsearch:0.2-incubating-SNAPSHOT.jar java -cp twitter-userstream-elasticsearch-0.2-incubating-SNAPSHOT.jar -Dconfig.file=http://<location_of_config_file>.json org.apache.streams.example.twitter.TwitterUserstreamElasticsearch` http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-userstream-elasticsearch/src/site/resources/TwitterUserstreamElasticsearch.dot ---------------------------------------------------------------------- diff --git a/local/twitter-userstream-elasticsearch/src/site/resources/TwitterUserstreamElasticsearch.dot b/local/twitter-userstream-elasticsearch/src/site/resources/TwitterUserstreamElasticsearch.dot new file mode 100644 index 0000000..c876176 --- /dev/null +++ b/local/twitter-userstream-elasticsearch/src/site/resources/TwitterUserstreamElasticsearch.dot @@ -0,0 +1,28 @@ +digraph g { + + //providers + TwitterStreamProvider [label="TwitterStreamProvider",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java"]; + + //processors + ActivityConverterProcessor [label="ActivityConverterProcessor",shape=box,URL="https://github.com/apache/incubator-streams/blob/master/streams-components/streams-converters/src/main/java/org/apache/streams/converter/ActivityConverterProcessor.java"]; + DeleteOnlyProcessor [label="VerbDefinitionKeepFilter (verb:delete)",shape=box,URL="https://github.com/apache/incubator-streams/blob/master/streams-components/streams-filters/src/main/java/org/apache/streams/filters/VerbDefinitionKeepFilter.java"]; + NoDeletesProcessor [label="VerbDefinitionDropFilter (verb:delete)",shape=box,URL="https://github.com/apache/incubator-streams/blob/master/streams-components/streams-filters/src/main/java/org/apache/streams/filters/VerbDefinitionDropFilter.java"]; + SetDeleteIdProcessor [label="SetDeleteIdProcessor (verb:post)",shape=box,URL="https://github.com/apache/incubator-streams-examples/blob/master/local/twitter-userstream-elasticsearch/src/main/java/org/apache/streams/twitter/example/TwitterUserstreamElasticsearch.java"]; + + //persisters + ElasticsearchPersistWriter [label="ElasticsearchPersistWriter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java"]; + ElasticsearchPersistDeleter [label="ElasticsearchPersistDeleter",shape=ellipse,URL="https://github.com/apache/incubator-streams/blob/master/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistDeleter.java"]; + + //data + es [label="es://{index}/{type}",shape=box]; + + //stream + TwitterStreamProvider -> ActivityConverterProcessor [label="ObjectNode"]; + ActivityConverterProcessor -> DeleteOnlyProcessor [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"]; + ActivityConverterProcessor -> NoDeletesProcessor [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"]; + DeleteOnlyProcessor -> SetDeleteIdProcessor [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"]; + NoDeletesProcessor -> ElasticsearchPersistWriter [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"]; + ElasticsearchPersistWriter -> es [label="Activity",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/activity.json"]; + SetDeleteIdProcessor -> ElasticsearchPersistDeleter [label="Delete",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/verbs/delete.json"]; + ElasticsearchPersistDeleter -> es [label="Delete",URL="https://github.com/apache/incubator-streams/blob/master/streams-pojo/src/main/jsonschema/org/apache/streams/pojo/json/delete.json"]; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-userstream-elasticsearch/src/site/resources/TwitterUserstreamElasticsearch.dot.png ---------------------------------------------------------------------- diff --git a/local/twitter-userstream-elasticsearch/src/site/resources/TwitterUserstreamElasticsearch.dot.png b/local/twitter-userstream-elasticsearch/src/site/resources/TwitterUserstreamElasticsearch.dot.png new file mode 100644 index 0000000..6e8713f Binary files /dev/null and b/local/twitter-userstream-elasticsearch/src/site/resources/TwitterUserstreamElasticsearch.dot.png differ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/107852b2/local/twitter-userstream-elasticsearch/src/site/resources/application.conf ---------------------------------------------------------------------- diff --git a/local/twitter-userstream-elasticsearch/src/site/resources/application.conf b/local/twitter-userstream-elasticsearch/src/site/resources/application.conf new file mode 100644 index 0000000..50d48b5 --- /dev/null +++ b/local/twitter-userstream-elasticsearch/src/site/resources/application.conf @@ -0,0 +1,22 @@ +twitter { + endpoint = "userstream" + oauth { + consumerKey = "" + consumerSecret = "" + accessToken = "" + accessTokenSecret = "" + } + follow = [ +â â â â â â +â â ] +} +elasticsearch { + hosts = [ + localhost + ] + port = 9300 + clusterName = elasticsearch + index = userstream_activity + type = activity + batchSize = 1 +}