twitter-follow-graph docker and twitter IT
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/commit/97fca1ac Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/tree/97fca1ac Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/diff/97fca1ac Branch: refs/heads/master Commit: 97fca1ac5a3ead3aa4e00b014a99c441b312e69e Parents: 476d356 Author: Steve Blackmon @steveblackmon <sblack...@apache.org> Authored: Mon Oct 10 17:26:22 2016 -0500 Committer: Steve Blackmon @steveblackmon <sblack...@apache.org> Committed: Mon Oct 10 17:26:22 2016 -0500 ---------------------------------------------------------------------- local/twitter-follow-graph/pom.xml | 78 ++++++++++++++++++- .../example/graph/TwitterFollowGraph.java | 33 ++++++-- .../TwitterFollowGraphConfiguration.json | 2 +- .../src/site/markdown/index.md | 25 +++++++ .../twitter/example/TwitterFollowGraphIT.java | 79 ++++++++++++++++++++ .../test/resources/TwitterFollowGraphIT.conf | 28 +++++++ 6 files changed, 233 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/97fca1ac/local/twitter-follow-graph/pom.xml ---------------------------------------------------------------------- diff --git a/local/twitter-follow-graph/pom.xml b/local/twitter-follow-graph/pom.xml index 9bf980d..4ce6a64 100644 --- a/local/twitter-follow-graph/pom.xml +++ b/local/twitter-follow-graph/pom.xml @@ -236,11 +236,81 @@ </execution> </executions> </plugin> - <plugin> - <groupId>io.fabric8</groupId> - <artifactId>docker-maven-plugin</artifactId> - </plugin> </plugins> </build> + <profiles> + <profile> + <id>dockerITs</id> + <activation> + <activeByDefault>false</activeByDefault> + <property> + <name>skipITs</name> + <value>false</value> + </property> + </activation> + <build> + <plugins> + <plugin> + <groupId>io.fabric8</groupId> + <artifactId>docker-maven-plugin</artifactId> + <version>${docker.plugin.version}</version> + <configuration combine.self="override"> + <watchInterval>500</watchInterval> + <logDate>default</logDate> + <verbose>true</verbose> + <autoPull>on</autoPull> + <images> + <image> + <name>neo4j</name> + <alias>graph</alias> + <run> + <env> + <NEO4J_AUTH>none</NEO4J_AUTH> + </env> + <namingStrategy>none</namingStrategy> + <ports> + <port>${graph.http.host}:${graph.http.port}:7474</port> + <port>${graph.tcp.host}:${graph.tcp.port}:7687</port> + </ports> + <portPropertyFile>graph.properties</portPropertyFile> + <wait> + <log>graph startup</log> + <http> + <url>http://${graph.http.host}:${graph.http.port}</url> + <method>GET</method> + <status>200</status> + </http> + <time>20000</time> + <kill>1000</kill> + <shutdown>500</shutdown> + <!--<tcp>--> + <!--<host>${es.transport.host}</host>--> + <!--<ports>--> + <!--<port>${es.transport.port}</port>--> + <!--</ports>--> + <!--</tcp>--> + </wait> + <log> + <enabled>true</enabled> + <date>default</date> + <color>cyan</color> + </log> + </run> + <watch> + <mode>none</mode> + </watch> + </image> + + </images> + </configuration> + + </plugin> + + </plugins> + </build> + + </profile> + </profiles> + </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/97fca1ac/local/twitter-follow-graph/src/main/java/org/apache/streams/example/graph/TwitterFollowGraph.java ---------------------------------------------------------------------- diff --git a/local/twitter-follow-graph/src/main/java/org/apache/streams/example/graph/TwitterFollowGraph.java b/local/twitter-follow-graph/src/main/java/org/apache/streams/example/graph/TwitterFollowGraph.java index e88f299..11c52bb 100644 --- a/local/twitter-follow-graph/src/main/java/org/apache/streams/example/graph/TwitterFollowGraph.java +++ b/local/twitter-follow-graph/src/main/java/org/apache/streams/example/graph/TwitterFollowGraph.java @@ -47,19 +47,23 @@ import java.util.Map; * Collects friend and follow connections for a set of twitter users and builds a graph * database in neo4j. */ -public class TwitterFollowGraph { +public class TwitterFollowGraph implements Runnable { private final static Logger LOGGER = LoggerFactory.getLogger(TwitterFollowGraph.class); - public static void main(String[] args) { + TwitterFollowGraphConfiguration config; - LOGGER.info(StreamsConfigurator.config.toString()); + public TwitterFollowGraph() { + this(new ComponentConfigurator<>(TwitterFollowGraphConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig())); + } - StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(); + public TwitterFollowGraph(TwitterFollowGraphConfiguration config) { + this.config = config; + } - TwitterFollowingGraphConfiguration configuration = new ComponentConfigurator<>(TwitterFollowingGraphConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig()); + public void run() { - TwitterFollowingConfiguration twitterFollowingConfiguration = configuration.getTwitter(); + TwitterFollowingConfiguration twitterFollowingConfiguration = config.getTwitter(); TwitterFollowingProvider followingProvider = new TwitterFollowingProvider(twitterFollowingConfiguration); TypeConverterProcessor converter = new TypeConverterProcessor(String.class); @@ -69,7 +73,7 @@ public class TwitterFollowGraph { .withConverters(Lists.newArrayList((ActivityConverter) new TwitterFollowActivityConverter())); ActivityConverterProcessor activity = new ActivityConverterProcessor(activityConverterProcessorConfiguration); - GraphHttpConfiguration graphWriterConfiguration = configuration.getGraph(); + GraphHttpConfiguration graphWriterConfiguration = config.getGraph(); GraphHttpPersistWriter graphPersistWriter = new GraphHttpPersistWriter(graphWriterConfiguration); StreamBuilder builder = new LocalStreamBuilder(); @@ -81,4 +85,19 @@ public class TwitterFollowGraph { builder.start(); } + public static void main(String[] args) { + + LOGGER.info(StreamsConfigurator.config.toString()); + + TwitterFollowGraph stream = new TwitterFollowGraph(); + + stream.run(); + + LOGGER.info(StreamsConfigurator.config.toString()); + + StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(); + + + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/97fca1ac/local/twitter-follow-graph/src/main/jsonschema/TwitterFollowGraphConfiguration.json ---------------------------------------------------------------------- diff --git a/local/twitter-follow-graph/src/main/jsonschema/TwitterFollowGraphConfiguration.json b/local/twitter-follow-graph/src/main/jsonschema/TwitterFollowGraphConfiguration.json index 129be4c..f9c4ac1 100644 --- a/local/twitter-follow-graph/src/main/jsonschema/TwitterFollowGraphConfiguration.json +++ b/local/twitter-follow-graph/src/main/jsonschema/TwitterFollowGraphConfiguration.json @@ -4,7 +4,7 @@ "http://www.apache.org/licenses/LICENSE-2.0" ], "type": "object", - "javaType" : "org.apache.streams.example.graph.TwitterFollowingGraphConfiguration", + "javaType" : "org.apache.streams.example.graph.TwitterFollowGraphConfiguration", "javaInterfaces": ["java.io.Serializable"], "properties": { "twitter": { "javaType": "org.apache.streams.twitter.TwitterFollowingConfiguration", "type": "object", "required": true }, http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/97fca1ac/local/twitter-follow-graph/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/local/twitter-follow-graph/src/site/markdown/index.md b/local/twitter-follow-graph/src/site/markdown/index.md index b8494cd..3991688 100644 --- a/local/twitter-follow-graph/src/site/markdown/index.md +++ b/local/twitter-follow-graph/src/site/markdown/index.md @@ -30,6 +30,31 @@ Build: mvn clean package verify +Test: +----- +Create a local file `application.conf` with valid twitter credentials + + twitter { + oauth { + consumerKey = "" + consumerSecret = "" + accessToken = "" + accessTokenSecret = "" + } + } + +Start up neo4j with docker: + + mvn -PdockerITs docker:start + +Build with integration testing enabled, using your credentials + + mvn clean test verify -DskipITs=false -DargLine="-Dconfig.file=`pwd`/application.conf" + +Shutdown neo4j when finished: + + mvn -PdockerITs docker:stop + Run (Local): ------------ http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/97fca1ac/local/twitter-follow-graph/src/test/java/org/apache/streams/twitter/example/TwitterFollowGraphIT.java ---------------------------------------------------------------------- diff --git a/local/twitter-follow-graph/src/test/java/org/apache/streams/twitter/example/TwitterFollowGraphIT.java b/local/twitter-follow-graph/src/test/java/org/apache/streams/twitter/example/TwitterFollowGraphIT.java new file mode 100644 index 0000000..c5254fe --- /dev/null +++ b/local/twitter-follow-graph/src/test/java/org/apache/streams/twitter/example/TwitterFollowGraphIT.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.streams.twitter.example; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigParseOptions; +import org.apache.streams.config.ComponentConfigurator; +import org.apache.streams.config.StreamsConfiguration; +import org.apache.streams.config.StreamsConfigurator; +import org.apache.streams.example.graph.TwitterFollowGraph; +import org.apache.streams.example.graph.TwitterFollowGraphConfiguration; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Properties; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; + +/** + * Example stream that populates elasticsearch with activities from twitter userstream in real-time + */ +public class TwitterFollowGraphIT { + + private final static Logger LOGGER = LoggerFactory.getLogger(TwitterFollowGraphIT.class); + + protected TwitterFollowGraphConfiguration testConfiguration; + + private int count = 0; + + @Before + public void prepareTest() throws Exception { + + Config reference = ConfigFactory.load(); + File conf_file = new File("target/test-classes/TwitterFollowGraphIT.conf"); + assert(conf_file.exists()); + Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); + Properties graph_properties = new Properties(); + InputStream graph_stream = new FileInputStream("graph.properties"); + graph_properties.load(graph_stream); + Config graphProps = ConfigFactory.parseProperties(graph_properties); + Config typesafe = testResourceConfig.withFallback(graphProps).withFallback(reference).resolve(); + StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe); + testConfiguration = new ComponentConfigurator<>(TwitterFollowGraphConfiguration.class).detectConfiguration(typesafe); + + } + + @Test + public void testTwitterFollowGraph() throws Exception { + + TwitterFollowGraph stream = new TwitterFollowGraph(testConfiguration); + + stream.run(); + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/97fca1ac/local/twitter-follow-graph/src/test/resources/TwitterFollowGraphIT.conf ---------------------------------------------------------------------- diff --git a/local/twitter-follow-graph/src/test/resources/TwitterFollowGraphIT.conf b/local/twitter-follow-graph/src/test/resources/TwitterFollowGraphIT.conf new file mode 100644 index 0000000..ecd4fd4 --- /dev/null +++ b/local/twitter-follow-graph/src/test/resources/TwitterFollowGraphIT.conf @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +twitter { + endpoint = "friends" + info = [ + 18055613 + ] + twitter.max_items = 1000 +} +graph { + hostname = ${graph.http.host} + port = ${graph.http.port} + type = "neo4j" + graph = "data" +} \ No newline at end of file