resolves STREAMS-410
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/ce7aa24d Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/ce7aa24d Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/ce7aa24d Branch: refs/heads/invalid_headers Commit: ce7aa24dab5a417a0470a0ce8e0f7cd2f3520271 Parents: a726b3c Author: Elias Ponvert <eponv...@gmail.com> Authored: Tue Oct 11 17:02:40 2016 -0500 Committer: Elias Ponvert <eponv...@gmail.com> Committed: Tue Oct 11 17:02:40 2016 -0500 ---------------------------------------------------------------------- pom.xml | 1 - .../streams-processor-lucene/pom.xml | 163 --------- .../lucene/LuceneSimpleTaggingProcessor.java | 355 ------------------- .../lucene/LuceneTaggerConfiguration.json | 29 -- .../lucene/TestLucenSimpleTaggingProcessor.java | 171 --------- .../src/test/resources/TestTags.tsv | 2 - streams-contrib/streams-processor-tika/pom.xml | 162 --------- .../org/apache/streams/tika/CategoryParser.java | 114 ------ .../org/apache/streams/tika/TikaProcessor.java | 139 -------- .../apache/streams/tika/BoilerPipeArticle.json | 76 ---- .../apache/streams/tika/TestCategoryParser.java | 30 -- .../apache/streams/tika/TestTikaProcessor.java | 29 -- .../activity-consumer/osgi.bnd | 9 - .../activity-consumer/pom.xml | 171 --------- .../activityconsumer/ActivityConsumer.java | 39 -- .../ActivityConsumerWarehouse.java | 31 -- .../impl/ActivityConsumerWarehouseImpl.java | 57 --- .../impl/PushActivityConsumer.java | 106 ------ .../spring/activity-consumer-context.xml | 28 -- .../META-INF/spring/activity-consumer-osgi.xml | 29 -- .../src/main/resources/readme.txt | 1 - .../activity-registration/osgi.bnd | 9 - .../activity-registration/pom.xml | 130 ------- .../ActivityPublisherRegistration.java | 29 -- .../ActivityStreamsSubscriberRegistration.java | 29 -- .../impl/ActivityPublisherRegistrationImpl.java | 67 ---- ...tivityStreamsSubscriberRegistrationImpl.java | 67 ---- .../activity-registration-context-osgi.xml | 30 -- .../spring/activity-registration-context.xml | 29 -- .../src/main/resources/readme.txt | 1 - .../activity-subscriber/osgi.bnd | 9 - .../activity-subscriber/pom.xml | 134 ------- .../ActivityStreamsSubscriber.java | 38 -- .../ActivityStreamsSubscriberWarehouse.java | 34 -- .../ActivityStreamsSubscription.java | 38 -- .../ActivityStreamsSubscriptionFilter.java | 34 -- .../ActivityStreamsSubscriptionOutput.java | 47 --- .../impl/ActivityStreamsSubscriberDelegate.java | 126 ------- .../ActivityStreamsSubscriberWarehouseImpl.java | 60 ---- ...yStreamsSubscriptionCassandraFilterImpl.java | 42 --- .../impl/ActivityStreamsSubscriptionImpl.java | 74 ---- ...vityStreamsSubscriptionLuceneFilterImpl.java | 43 --- .../spring/activity-subscriber-context.xml | 28 -- .../spring/activity-subscriber-osgi.xml | 29 -- .../src/main/resources/readme.txt | 1 - .../src/site/resources/readme.txt | 1 - streams-osgi-components/pom.xml | 43 --- streams-runtimes/pom.xml | 1 - .../jsonschema/SparkRuntimeConfiguration.json | 27 -- .../streams-runtime-webapp/README.md | 8 - streams-runtimes/streams-runtime-webapp/pom.xml | 250 ------------- .../aggregation/ActivityAggregator.java | 59 --- .../configuration/EipConfigurator.java | 201 ----------- .../ActivityPublisherRegistrationProcessor.java | 72 ---- ...yStreamsSubscriberRegistrationProcessor.java | 93 ----- .../routers/ActivityConsumerRouteBuilder.java | 32 -- .../ActivityStreamsSubscriberRouteBuilder.java | 32 -- .../routers/impl/ActivityConsumerRouter.java | 144 -------- .../impl/ActivityStreamsSubscriberRouter.java | 141 -------- .../messaging/service/ActivityService.java | 31 -- .../messaging/service/SubscriptionService.java | 29 -- .../service/impl/CassandraActivityService.java | 96 ----- .../impl/CassandraSubscriptionService.java | 43 --- .../META-INF/spring/propertiesLoader.xml | 35 -- .../spring/streams-eip-applicationContext.xml | 113 ------ .../streams-eip-osgi-component-import.xml | 38 -- .../META-INF/spring/streamsCamelContext.xml | 96 ----- .../main/resources/META-INF/streams.properties | 41 --- .../impl/CassandraActivityServiceTest.java | 86 ----- 69 files changed, 4582 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index d51b25f..466e94b 100644 --- a/pom.xml +++ b/pom.xml @@ -174,7 +174,6 @@ <module>streams-contrib</module> <module>streams-components</module> <module>streams-monitoring</module> - <!--<module>streams-osgi-components</module>--> <module>streams-pojo</module> <module>streams-pojo-extensions</module> <module>streams-runtimes</module> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-contrib/streams-processor-lucene/pom.xml ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-processor-lucene/pom.xml b/streams-contrib/streams-processor-lucene/pom.xml deleted file mode 100644 index d8dba13..0000000 --- a/streams-contrib/streams-processor-lucene/pom.xml +++ /dev/null @@ -1,163 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - - <modelVersion>4.0.0</modelVersion> - <artifactId>streams-processor-lucene</artifactId> - - <parent> - <groupId>org.apache.streams</groupId> - <artifactId>streams-contrib</artifactId> - <version>0.1-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - - <properties> - <!-- This is the Version Number for the Apache Streams Project --> - <streams.version>0.1-SNAPSHOT</streams.version> - </properties> - <dependencies> - <dependency> - <groupId>org.apache.streams</groupId> - <artifactId>streams-config</artifactId> - <version>${streams.version}</version> - </dependency> - <dependency> - <groupId>org.apache.streams</groupId> - <artifactId>streams-core</artifactId> - <version>${streams.version}</version> - </dependency> - <dependency> - <groupId>org.apache.streams</groupId> - <artifactId>streams-pojo</artifactId> - <version>${streams.version}</version> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - </dependency> - <dependency> - <groupId>org.jsonschema2pojo</groupId> - <artifactId>jsonschema2pojo-core</artifactId> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - </dependency> - <dependency> - <groupId>com.jayway.jsonpath</groupId> - <artifactId>json-path</artifactId> - </dependency> - <dependency> - <groupId>com.jayway.jsonpath</groupId> - <artifactId>json-path-assert</artifactId> - </dependency> - <dependency> - <groupId>org.apache.streams</groupId> - <artifactId>streams-testing</artifactId> - <version>${project.version}</version> - <scope>test</scope> - <type>test-jar</type> - </dependency> - </dependencies> - - <build> - <sourceDirectory>src/main/java</sourceDirectory> - <testSourceDirectory>src/test/java</testSourceDirectory> - <resources> - <resource> - <directory>src/main/resources</directory> - </resource> - </resources> - <testResources> - <testResource> - <directory>src/test/resources</directory> - </testResource> - </testResources> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <executions> - <execution> - <id>add-source</id> - <phase>generate-sources</phase> - <goals> - <goal>add-source</goal> - </goals> - <configuration> - <sources> - <source>target/generated-sources/jsonschema2pojo/**/*.java</source> - </sources> - </configuration> - </execution> - <execution> - <id>add-source-jaxb2</id> - <phase>generate-sources</phase> - <goals> - <goal>add-source</goal> - </goals> - <configuration> - <sources> - <source>target/generated-sources/jaxb2</source> - </sources> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.jsonschema2pojo</groupId> - <artifactId>jsonschema2pojo-maven-plugin</artifactId> - <configuration> - <addCompileSourceRoot>true</addCompileSourceRoot> - <generateBuilders>true</generateBuilders> - <sourcePaths> - <sourcePath>${project.basedir}/src/main/jsonschema/org/apache/streams/lucene/LuceneTaggerConfiguration.json</sourcePath> - </sourcePaths> - <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory> - <targetPackage>org.apache.streams.lucene</targetPackage> - <useLongIntegers>true</useLongIntegers> - <useJodaDates>true</useJodaDates> - </configuration> - <executions> - <execution> - <goals> - <goal>generate</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-contrib/streams-processor-lucene/src/main/java/org/apache/streams/lucene/LuceneSimpleTaggingProcessor.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-processor-lucene/src/main/java/org/apache/streams/lucene/LuceneSimpleTaggingProcessor.java b/streams-contrib/streams-processor-lucene/src/main/java/org/apache/streams/lucene/LuceneSimpleTaggingProcessor.java deleted file mode 100644 index 2595afd..0000000 --- a/streams-contrib/streams-processor-lucene/src/main/java/org/apache/streams/lucene/LuceneSimpleTaggingProcessor.java +++ /dev/null @@ -1,355 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.streams.lucene; - -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import com.google.inject.Injector; -import com.jayway.jsonpath.InvalidPathException; -import com.jayway.jsonpath.JsonPath; -import com.w2olabs.core.graph.CommunityRepository; -import com.w2olabs.core.graph.CommunityRepositoryResolver; -import com.w2olabs.core.graph.entities.Entity; -import com.w2olabs.streams.pojo.W2OActivity; -import com.w2olabs.util.guice.GuiceInjector; -import com.w2olabs.util.tagging.SimpleVerbatim; -import com.w2olabs.util.tagging.engines.international.LanguageTag; -import com.w2olabs.util.tagging.engines.international.TaggingEngineLanguage; -import org.apache.avro.data.Json; -import org.apache.commons.lang3.StringUtils; -import org.apache.streams.core.StreamsDatum; -import org.apache.streams.core.StreamsProcessor; -import org.apache.streams.jackson.StreamsJacksonMapper; -import org.apache.streams.pojo.json.Activity; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.Serializable; -import java.util.*; -import java.util.concurrent.LinkedBlockingQueue; - -/** - * References: - * Some helpful references to help - * Purpose URL - * ------------- ---------------------------------------------------------------- - * [Status Codes] http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html - * [Test Cases] http://greenbytes.de/tech/tc/httpredirects/ - * [t.co behavior] https://dev.twitter.com/docs/tco-redirection-behavior - */ - -public class LuceneSimpleTaggingProcessor implements StreamsProcessor -{ - private final static String STREAMS_ID = "LuceneSimpleTaggingProcessor"; - - private final static Logger LOGGER = LoggerFactory.getLogger(LuceneSimpleTaggingProcessor.class); - - private ObjectMapper mapper; - - private Queue<StreamsDatum> inQueue; - private Queue<StreamsDatum> outQueue; - - private String community; - private JsonPath[] textPaths; - private String[] jsonPathsToText; - private List<LanguageTag> tags; - private String metaDataKey; - - private static TaggingEngineLanguage<LanguageTag, SimpleVerbatim> taggingEngine; - - /** - * Constructor for a tagging processor that will operate on the document of StreamsDatum at the paths noted by - * the json paths paramter - * @param community - * @param jsonPathsToText - */ - public LuceneSimpleTaggingProcessor(String community, String[] jsonPathsToText) { - this(community, jsonPathsToText, null, null); - } - - /** - * Constructor for a tagging processor that will operate on some meta data field of the StreamsDatum indicated by the - * meta data key. The data in the meta data field is still expected to be json or able to be converted to json or - * a list of such data. - * @param community - * @param jsonPathsToText - * @param metaDataKey - */ - public LuceneSimpleTaggingProcessor(String community, String[] jsonPathsToText, String metaDataKey) { - this(community, jsonPathsToText, metaDataKey, null); - } - - - /** - * For testing purposes. LanguageTag are not serializable and need to be set through the community resolver in production. - * @param community - * @param jsonPathsToText - * @param tags - */ - public LuceneSimpleTaggingProcessor(String community, String[] jsonPathsToText, String metaDataKey, List<LanguageTag> tags) { - this.community = community; - this.jsonPathsToText = jsonPathsToText; - this.tags = tags; - this.metaDataKey = metaDataKey; - verifyJsonPathstoText(this.jsonPathsToText); - } - -// public LuceneSimpleTaggingProcessor(Queue<StreamsDatum> inQueue) { -// this.inQueue = inQueue; -// this.outQueue = new LinkedBlockingQueue<StreamsDatum>(); -// } - -// public void stop() { -// -// } - - public String getCommunity() { - return community; - } - - public void setCommunity(String community) { - this.community = community; - } - - @Override - public List<StreamsDatum> process(StreamsDatum entry) { - - - LOGGER.debug("{} processing {}", STREAMS_ID, entry.getDocument().getClass()); - - List<StreamsDatum> result = Lists.newArrayList(); - - List<String> jsons = Lists.newLinkedList(); - ObjectNode node; - // first check for valid json - if(this.metaDataKey == null) - jsons.add(getJson(entry.getDocument())); - else - getMetaDataJsons(entry, jsons); - - for(String json : jsons) { - try { - node = (ObjectNode) mapper.readTree(json); - } catch (IOException e) { - e.printStackTrace(); - return result; - } - - List<SimpleVerbatim> verbatimList = convertEntryToWorkUnit(json); - Map<SimpleVerbatim, List<LanguageTag>> objectTags = taggingEngine.findMatches(verbatimList); - Set<String> tagSet = Sets.newHashSet(); - for( List<LanguageTag> fieldtags : objectTags.values() ) { - for( LanguageTag tag : fieldtags ) { - tagSet.add(tag.getTag()); - } - } - - ArrayNode tagArray = JsonNodeFactory.instance.arrayNode(); - Set<String> tags = Sets.newHashSet(); - for( String tag : tagSet ) { - if(tags.add(tag)){ - tagArray.add(tag); - } - } - - - - // need utility methods for get / create specific node - ObjectNode extensions = (ObjectNode) node.get("extensions"); - if(extensions == null) { - extensions = JsonNodeFactory.instance.objectNode(); - node.put("extensions", extensions); - } - ObjectNode w2o = (ObjectNode) extensions.get("w2o"); - if(w2o == null) { - w2o = JsonNodeFactory.instance.objectNode(); - extensions.put("w2o", w2o); - } - w2o.put("tags", tagArray); - w2o.put("contentTags", tagArray); - if(entry.getDocument() instanceof W2OActivity) { - entry.setDocument(mapper.convertValue(node, W2OActivity.class)); - } - else if(entry.getDocument() instanceof Activity) { - entry.setDocument(mapper.convertValue(node, Activity.class)); - } - else if(entry.getDocument() instanceof String) { - try { - entry.setDocument(mapper.writeValueAsString(node)); - } catch (JsonProcessingException jpe) { - LOGGER.error("Exception while converting ObjectNode to string. Outputing as ObjectNode. {}", jpe); - entry.setDocument(node); - } - } - else { - entry.setDocument(node); - } - result.add(entry); - } - return result; - } - - private void getMetaDataJsons(StreamsDatum datum, List<String> jsons) { - if(datum.getMetadata() == null) - return; - Object obj = datum.getMetadata().get(this.metaDataKey); - if(obj == null) { - LOGGER.debug("Object at key={} was NULL.", this.metaDataKey); - return; - } - if(obj instanceof List) { - List list = (List) obj; - String json; - for(Object o : list) { - json = getJson(o); - if(json != null) { - jsons.add(json); - } - } - } - else { - String json = getJson(obj); - if(json != null) { - jsons.add(json); - } - } - } - - private String getJson(Object object) { - String json = null; - if( object instanceof String ) { - json = (String) object; - } else if(object instanceof Activity){ - try { - json = mapper.writeValueAsString(object); - } catch (JsonProcessingException jpe) { - json = null; - LOGGER.error("Failed to convert Activity to String : {}", jpe); - } - } else { - ObjectNode node = (ObjectNode) object; - json = node.asText(); - } - return json; - } - - @Override - public void prepare(Object o) { - mapper = StreamsJacksonMapper.getInstance(); - if(this.tags == null) { - resolver = injector.getInstance(CommunityRepositoryResolver.class); - repo = resolver.get(community); - List<Entity> entities = repo.getTaggableEntities(); - taggingEngine = new TaggingEngineLanguage<LanguageTag, SimpleVerbatim>(createLanguageTagsFromRexsterTags(entities)); - } - else { - taggingEngine = new TaggingEngineLanguage<LanguageTag, SimpleVerbatim>(this.tags); - } - compileTextJsonPaths(); - } - - @Override - public void cleanUp() { - - } - - private static List<LanguageTag> createLanguageTagsFromRexsterTags(List<Entity> graphTags) { - List<LanguageTag> result = new ArrayList<LanguageTag>(graphTags.size()); - LOGGER.info("Attempting to convert {} Graph tags into Language tags.", graphTags.size()); - String tagLang; - for(Entity tag : graphTags) { -// net.sf.json.JSONObject json = null; - try { -// json = (net.sf.json.JSONObject) tag; - tagLang = tag.getAdditionalProperties().get("language") == null ? "en" : tag.getAdditionalProperties().get("language").toString(); - } catch (Exception e) { - tagLang = "en"; - } - LOGGER.debug("Tag : {}\n{}\n{}", new String[] {tag.getIdentifier(),tag.getQuery(),tagLang}); - if(TaggingEngineLanguage.SUPPORTED.contains(tagLang)) { - try { - result.add(new LanguageTag(tag.getIdentifier(),tag.getQuery(), tagLang)); - } catch (Exception e) { - e.printStackTrace(); - //result.add(new LanguageTag(tag.getIdentifier(), tag.getQuery(), tagLang)); - } - } - else { - LOGGER.warn("Attempted to load a tag for Language, {}, but that language is not currently supported. SimpleTag is being ignored!"); - } - } - LOGGER.info("Loaded {} Language Tags. {} tags were not supported.", result.size(), graphTags.size() - result.size()); - return result; - } - - // Does basic verification that paths are not null and in JsonPath syntax. - private String[] verifyJsonPathstoText(String[] jsonPathsToText) { - RuntimeException e = null; - for(String path : jsonPathsToText) { - if(StringUtils.isEmpty(path) || !path.matches("[$][.][a-zA-z0-9.]+")) { - LOGGER.error("Invalid JsonPath path : {}", path); - e = new RuntimeException("Invalid JsonPath paths!"); - } - } - if(e != null) - throw e; - return jsonPathsToText; - } - - // Compiles jsonPathToText to JsonPath - private void compileTextJsonPaths() { - this.textPaths = new JsonPath[this.jsonPathsToText.length]; - for(int i=0; i < this.jsonPathsToText.length; ++i) { - this.textPaths[i] = JsonPath.compile(this.jsonPathsToText[i]); - } - } - - private List<SimpleVerbatim> convertEntryToWorkUnit(String json) { - List<SimpleVerbatim> textFields = new ArrayList<SimpleVerbatim>(); - for(JsonPath path : this.textPaths) { - try { - Object pathObject = path.read(json); - if( pathObject instanceof String ) - textFields.add(new SimpleVerbatim((String) pathObject) ); - else if( pathObject instanceof List ) { - List<String> pathObjectList = (List<String>) pathObject; - for( String pathItem : pathObjectList ) { - textFields.add(new SimpleVerbatim(pathItem) ); - } - } - } catch( InvalidPathException x ) { - LOGGER.debug("{}: {}", x.getMessage(), path.getPath()); - } catch( ClassCastException x ) { - LOGGER.warn(x.getMessage()); - } - } - - if(textFields.size() == 0) - return null; - - return textFields; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-contrib/streams-processor-lucene/src/main/jsonschema/org/apache/streams/lucene/LuceneTaggerConfiguration.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-processor-lucene/src/main/jsonschema/org/apache/streams/lucene/LuceneTaggerConfiguration.json b/streams-contrib/streams-processor-lucene/src/main/jsonschema/org/apache/streams/lucene/LuceneTaggerConfiguration.json deleted file mode 100644 index 048503e..0000000 --- a/streams-contrib/streams-processor-lucene/src/main/jsonschema/org/apache/streams/lucene/LuceneTaggerConfiguration.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "type": "object", - "javaType" : "org.apache.streams.lucene.LuceneTaggerConfiguration", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "tags": { - "type": "array", - "items": { - "tag": { - "type": "string", - "description": "Tag Identifier" - }, - "query": { - "type": "string", - "description": "Lucene Query" - }, - "language": { - "type": "string", - "description": "Language" - } - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-contrib/streams-processor-lucene/src/test/java/org/apache/streams/lucene/TestLucenSimpleTaggingProcessor.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-processor-lucene/src/test/java/org/apache/streams/lucene/TestLucenSimpleTaggingProcessor.java b/streams-contrib/streams-processor-lucene/src/test/java/org/apache/streams/lucene/TestLucenSimpleTaggingProcessor.java deleted file mode 100644 index 62355c5..0000000 --- a/streams-contrib/streams-processor-lucene/src/test/java/org/apache/streams/lucene/TestLucenSimpleTaggingProcessor.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.streams.lucene; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; -import com.jayway.jsonpath.JsonPath; -import com.w2olabs.streams.pojo.W2OActivity; -import com.w2olabs.util.tagging.engines.international.LanguageTag; -import org.apache.streams.core.StreamsDatum; -import org.apache.streams.pojo.json.Activity; -import org.junit.BeforeClass; -import org.junit.Test; -import static org.junit.Assert.*; - -import java.util.List; -import java.util.Scanner; - -/** - * Created by rebanks on 3/18/14. - */ -public class TestLucenSimpleTaggingProcessor { - - private static final String ACTIVITY_JSON = "{\"id\":\"id:twitter:post:410898682381615105\",\"actor\":{\"id\":\"91407775\",\"displayName\":\"winchester_ky\",\"attachments\":[],\"upstreamDuplicates\":[],\"downstreamDuplicates\":[]},\"verb\":\"post\",\"object\":{\"id\":\"id:twitter:410898682381615105\",\"objectType\":\"tweet\",\"attachments\":[],\"upstreamDuplicates\":[],\"downstreamDuplicates\":[]},\"published\":1386800854000,\"provider\":{\"id\":\"id:providers:twitter\",\"attachments\":[],\"upstreamDuplicates\":[],\"downstreamDuplicates\":[]},\"title\":\"\",\"content\":\"Men's Basketball baseball soccer Single-Game Tickets Available - A limited number of tickets remain for Kentucky's upcoming men's ... http://t.co/SH5YZGpdRx\",\"links\":[\"http://ow.ly/2C2XL1\"],\"extensions\":{\"twitter\":{\"created_at\":\"Wed Dec 11 22:27:34 +0000 2013\",\"id\":410898682381615105,\"id_str\":\"410898682381615105\",\"text\":\"Men's Basketball Single-Game Tickets Available - A limited number of t ickets remain for Kentucky's upcoming men's ... http://t.co/SH5YZGpdRx\",\"source\":\";<a href=\\\"http://www.hootsuite.com\\\"; rel=\\\"nofollow\\\">HootSuite</a>\",\"truncated\":false,\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":91407775,\"id_str\":\"91407775\",\"name\":\"Winchester, KY\",\"screen_name\":\"winchester_ky\",\"location\":\"\",\"url\":null,\"description\":null,\"protected\":false,\"followers_count\":136,\"friends_count\":0,\"listed_count\":1,\"created_at\":\"Fri Nov 20 19:29:02 +0000 2009\",\"favourites_count\":0,\"utc_offset\":null,\"time_zone\":null,\"geo_enabled\":false,\"verified\":false,\"statuses_count\":1793,\"lang\":\"en\",\"contributors_enabled\":false,\"is_translator\":false,\"profile_background_color\":\"C0DEED\",\"profile_background_image_url\":\"http://abs.twimg.com/images/themes/theme1/bg.png\",\"profile_background_image_url_ https\":\"https://abs.twimg.com/images/themes/theme1/bg.png\",\"profile_background_tile\":false,\"profile_image_url\":\"http://pbs.twimg.com/profile_images/613854495/winchester_sociallogo_normal.jpg\",\"profile_image_url_https\":\"https://pbs.twimg.com/profile_images/613854495/winchester_sociallogo_normal.jpg\",\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"symbols\":[],\"urls\":[{\"url\":\"http://t.co/SH5YZGpdRx\",\"expanded_url\":\"http://ow.ly/2C2XL1\",\"display_url\":\"ow.ly/2C2XL1\",\"indices\":[118,140]}],\"user_mentions\":[]},\"favorited\":false,\"retweeted\":false, \"possibly_sensitive\":false,\"filter_level\":\"medium\",\"lang\":\"en\"},\"location\":{\"id\":\"id:twitter:410898682381615105\",\"coordinates\":null}}}\n"; - private static final String W2O_ACTIVITY_JSON = "{\"extensions\":{\"w2o\":{\"provider\":\"twitter\",\"analyzer\":\"romance_analyzer\",\"tags\":[\"brand-cascade\",\"language-en\",\"country-ca\"],\"contentTags\":[],\"linkTags\":[],\"lang\":{\"primaryLanguage\":\"en\"}},\"twitter\":{\"retweetCount\":0,\"entities\":{\"symbols\":[],\"urls\":[{\"expanded_url\":\"http://bit.ly/hUmoRz\",\"indices\":[77,99],\"display_url\":\"bit.ly/hUmoRz\",\"url\":\"http://t.co/Ytn45Pbttk\"}],\"hashtags\":[{\"text\":\"SmurfsVillage\",\"indices\":[100,114]}],\"user_mentions\":[{\"id\":188075479,\"name\":\"Beeline Interactive\",\"indices\":[115,128],\"screen_name\":\"BeelineGames\",\"id_str\":\"188075479\"}]}},\"gnip\":{\"matching_rules\":[{\"tag\":\"cascade_CA_CA_en\"}],\"klout_score\":10,\"urls\":[{\"expanded_url\":\"https://itunes.apple.com/us/app/smurfs-village/id399648212?mt=8\",\"url\":\"http://t.co/Ytn45Pbttk\"}],\"klout_profile\":{\"topics\":[],\"klout_user_id\":\"257268143479895040\",\"link\":\"h ttp://klout.com/user/id/257268143479895040\"},\"language\":{\"value\":\"fr\"}}},\"id\":\"tag:search.twitter.com,2005:372802927385403392\",\"actor\":{\"id\":\"id:twitter.com:583891967\",\"image\":{\"url\":\"https://si0.twimg.com/sticky/default_profile_images/default_profile_1_normal.png\"},\"displayName\":\"Sabine Chappuis\",\"objectType\":\"person\",\"attachments\":[],\"upstreamDuplicates\":[],\"downstreamDuplicates\":[],\"twitterTimeZone\":\"Brussels\",\"friendsCount\":6,\"favoritesCount\":0,\"link\":\"http://www.twitter.com/spoffff\",\"postedTime\":\"2012-05-18T15:14:35.000Z\",\"links\":[{\"rel\":\"me\",\"href\":null}],\"listedCount\":0,\"languages\":[\"fr\"],\"verified\":false,\"utcOffset\":\"7200\",\"followersCount\":0,\"preferredUsername\":\"spoffff\",\"statusesCount\":87},\"verb\":\"post\",\"object\":{\"id\":\"object:search.twitter.com,2005:372802927385403392\",\"summary\":\"Le Grand Schtroumpf confirme que la cascade magique n'est \\\"Plus tr?�s loin.\\\" http://t.co/Ytn45P bttk #SmurfsVillage @BeelineGames\",\"objectType\":\"note\",\"attachments\":[],\"upstreamDuplicates\":[],\"downstreamDuplicates\":[],\"link\":\"http://twitter.com/spoffff/statuses/372802927385403392\",\"postedTime\":\"2013-08-28T19:28:38.000Z\"},\"published\":1377718118000,\"generator\":{\"id\":\"{link}\",\"displayName\":\"Smurfs' Village on iOS\",\"attachments\":[],\"upstreamDuplicates\":[],\"downstreamDuplicates\":[],\"link\":\"https://itunes.apple.com/us/app/smurfs-village/id399648212?mt=8&uo=4\"},\"provider\":{\"id\":\"{link}\",\"displayName\":\"Twitter\",\"objectType\":\"service\",\"attachments\":[],\"upstreamDuplicates\":[],\"downstreamDuplicates\":[],\"link\":\"http://www.twitter.com\"},\"content\":\"Le Grand Schtroumpf confirme soccer que la cascade magique n'est \\\"Plus tr?�s loin.\\\" http://t.co/Ytn45Pbttk #SmurfsVillage @BeelineGames\",\"links\":[],\"guid\":\"A8fccSz7rpKfDJY078VLyw==_201308\",\"link\":\"http://twitter.com/spoffff/statuses/372802927385403392\",\"posted Time\":\"2013-08-28T19:28:38.000Z\",\"objectType\":\"activity\",\"twitter_filter_level\":\"medium\"}\n"; - private static final String LINK_EXPANDER_JSON = "{\"body\":\"Analise baseball Coady W2O Lead, EMEA Twitter Linkedin\\nRyan Flinn Director, Earned Media Twitter Linkedin\\nAdam Cohen W2O Lead, Boston Twitter Linkedin\\nSarah Savage Managing Director, Healthcare Twitter Linkedin\\nCarolyn Wang Practice Lead, Healthcare Twitter Linkedin\\nKathy Keanini Group Director, Strategy Twitter Linkedin\\nRob Cronin Practice Lead, Healthcare Twitter Linkedin\\nAnalise Coady W2O Lead, EMEA Twitter Linkedin\\nRyan Flinn Director, Earned Media Twitter Linkedin\\nAdam Cohen W2O Lead, Boston Twitter Linkedin\\nSarah Savage Managing Director, Healthcare Twitter Linkedin\\nCarolyn Wang Practice Lead, Healthcare Twitter Linkedin\\nKathy Keanini Group Director, Strategy Twitter Linkedin\\nRob Cronin Practice Lead, Healthcare Twitter Linkedin\\nOur Thinkers\\nFull-on enterprise consulting, supercharged by the best analytics in the business.\\nProducts & Services\\nMedia\\nPaid. Earned. Shared. Owned. You name it, we either mastered it or just invented it.\\nthe w2o group\\n© W2O Group 2014\\n\",\"finalUrl\":\"http://www.wcgworld.com/\",\"locale\":null,\"twitterSite\":null,\"urlParts\":[\"www.wcgworld.com\"],\"twitterCreator\":null,\"finalStatusCode\":\"200\",\"author\":null,\"originalUrl\":\"http://www.wcgworld.com/\",\"title\":\"WCG World\",\"description\":\"Find out what makes The WCG Approach second to none\",\"redirects\":[],\"domain\":\"www.wcgworld.com\",\"wasRedirected\":false,\"facebookApp\":null,\"metaData\":{\"content-type\":\"text/html; charset=UTF-8\",\"viewport\":\"width=device-width, initial-scale=1\",\"title\":\"WCG World\",\"og:title\":\"WCG World\",\"og:description\":\"Find out what makes The WCG Approach second to none\",\"content-encoding\":\"UTF-8\",\"x-ua-compatible\":\"IE=edge\",\"og:site_name\":\"WCG World\",\"dc:title\":\"WCG World\",\"og:image\":\"http://www.wcgworld.com/assets/img/home_banner.png\",\"og:url\":\"http://www.wcgworld.com/\"},\"normalized Url\":\"www.wcgworld.com/\",\"keywords\":[\"keywords\",\"news_keywords\"],\"status\":\"SUCCESS\",\"isTracked\":false,\"medium\":null,\"facebookPage\":null,\"failure\":false,\"lastModifiedDate\":null,\"tookInMillis\":110,\"publishedDate\":null,\"siteStatus\":\"ERROR\",\"imageURL\":\"http://www.wcgworld.com/assets/img/home_banner.png\",\"plainText\":null}\n";; - - private static List<LanguageTag> tags; - private static ObjectMapper mapper; - - - @BeforeClass - public static void setUpTags() { - tags = Lists.newLinkedList(); - Scanner scanner = new Scanner(TestLucenSimpleTaggingProcessor.class.getResourceAsStream("/TestTags.tsv")); - while(scanner.hasNextLine()) { - String[] line = scanner.nextLine().split("\t"); - tags.add(new LanguageTag(line[0], line[1], "en")); - } - mapper = new ObjectMapper(); - } - - @Test - public void testSerializability() { - LuceneSimpleTaggingProcessor processor = new LuceneSimpleTaggingProcessor("testCommunity", new String[]{"test","path"}); - LuceneSimpleTaggingProcessor clone = SerializationUtils.clone(processor); - } - - @Test - public void testActivityJsonString() { - LuceneSimpleTaggingProcessor processor = new LuceneSimpleTaggingProcessor("test", new String[] {"$.content"}, null,tags); - processor.prepare(null); - List<StreamsDatum> datums = processor.process(new StreamsDatum(ACTIVITY_JSON)); - assertNotNull(datums); - assertEquals(1, datums.size()); - StreamsDatum datum = datums.get(0); - assertNotNull(datum); - assertNotNull(datum.getDocument()); - assertTrue(datum.getDocument() instanceof String); - String json = (String)datum.getDocument(); - List<String> tags = JsonPath.read(json, "$.extensions.w2o.tags"); - assertEquals(2, tags.size()); - assertTrue(tags.contains("baseball")); - assertTrue(tags.contains("soccer")); - tags = JsonPath.read(json, "$.extensions.w2o.contentTags"); - assertEquals(2, tags.size()); - assertTrue(tags.contains("baseball")); - assertTrue(tags.contains("soccer")); - } - - @Test - public void testW2OActivityJsonString() { - LuceneSimpleTaggingProcessor processor = new LuceneSimpleTaggingProcessor("test", new String[] {"$.content"}, null,tags); - processor.prepare(null); - List<StreamsDatum> datums = processor.process(new StreamsDatum(W2O_ACTIVITY_JSON)); - assertNotNull(datums); - assertEquals(1, datums.size()); - StreamsDatum datum = datums.get(0); - assertNotNull(datum); - assertNotNull(datum.getDocument()); - assertTrue(datum.getDocument() instanceof String); - String json = (String)datum.getDocument(); - List<String> tags = JsonPath.read(json, "$.extensions.w2o.tags"); - assertEquals(1, tags.size()); - assertTrue(tags.contains("soccer")); - tags = JsonPath.read(json, "$.extensions.w2o.contentTags"); - assertEquals(1, tags.size()); - assertTrue(tags.contains("soccer")); - } - - @Test - public void testLinkExpanderJsonString() { - LuceneSimpleTaggingProcessor processor = new LuceneSimpleTaggingProcessor("test", new String[] {"$.body"}, null, tags); - processor.prepare(null); - List<StreamsDatum> datums = processor.process(new StreamsDatum(LINK_EXPANDER_JSON)); - assertNotNull(datums); - assertEquals(1, datums.size()); - StreamsDatum datum = datums.get(0); - assertNotNull(datum); - assertNotNull(datum.getDocument()); - assertTrue(datum.getDocument() instanceof String); - String json = (String)datum.getDocument(); - List<String> tags = JsonPath.read(json, "$.extensions.w2o.tags"); - assertEquals(1, tags.size()); - assertTrue(tags.contains("baseball")); - tags = JsonPath.read(json, "$.extensions.w2o.contentTags"); - assertEquals(1, tags.size()); - assertTrue(tags.contains("baseball")); - } - - @Test - public void testActivityObject() throws Exception { - LuceneSimpleTaggingProcessor processor = new LuceneSimpleTaggingProcessor("test", new String[] {"$.content"}, null,tags); - processor.prepare(null); - List<StreamsDatum> datums = processor.process(new StreamsDatum(mapper.readValue(ACTIVITY_JSON, Activity.class))); - assertNotNull(datums); - assertEquals(1, datums.size()); - StreamsDatum datum = datums.get(0); - assertNotNull(datum); - assertNotNull(datum.getDocument()); - assertTrue(datum.getDocument() instanceof Activity); - String json = mapper.writeValueAsString(datum.getDocument()); - List<String> tags = JsonPath.read(json, "$.extensions.w2o.tags"); - assertEquals(2, tags.size()); - assertTrue(tags.contains("baseball")); - assertTrue(tags.contains("soccer")); - tags = JsonPath.read(json, "$.extensions.w2o.contentTags"); - assertEquals(2, tags.size()); - assertTrue(tags.contains("baseball")); - assertTrue(tags.contains("soccer")); - } - - @Test - public void testW2OActivityObject() throws Exception{ - LuceneSimpleTaggingProcessor processor = new LuceneSimpleTaggingProcessor("test", new String[] {"$.content"}, null,tags); - processor.prepare(null); - List<StreamsDatum> datums = processor.process(new StreamsDatum(mapper.readValue(W2O_ACTIVITY_JSON, W2OActivity.class))); - assertNotNull(datums); - assertEquals(1, datums.size()); - StreamsDatum datum = datums.get(0); - assertNotNull(datum); - assertNotNull(datum.getDocument()); - assertTrue(datum.getDocument() instanceof W2OActivity); - String json = (String) mapper.writeValueAsString(datum.getDocument()); - List<String> tags = JsonPath.read(json, "$.extensions.w2o.tags"); - assertEquals(1, tags.size()); - assertTrue(tags.contains("soccer")); - tags = JsonPath.read(json, "$.extensions.w2o.contentTags"); - assertEquals(1, tags.size()); - assertTrue(tags.contains("soccer")); - } - - - -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-contrib/streams-processor-lucene/src/test/resources/TestTags.tsv ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-processor-lucene/src/test/resources/TestTags.tsv b/streams-contrib/streams-processor-lucene/src/test/resources/TestTags.tsv deleted file mode 100644 index c6ed3a7..0000000 --- a/streams-contrib/streams-processor-lucene/src/test/resources/TestTags.tsv +++ /dev/null @@ -1,2 +0,0 @@ -baseball baseball OR homerun OR "home run" -soccer soccer \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-contrib/streams-processor-tika/pom.xml ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-processor-tika/pom.xml b/streams-contrib/streams-processor-tika/pom.xml deleted file mode 100644 index 9969e29..0000000 --- a/streams-contrib/streams-processor-tika/pom.xml +++ /dev/null @@ -1,162 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - - <modelVersion>4.0.0</modelVersion> - <artifactId>streams-processor-tika</artifactId> - - <parent> - <groupId>org.apache.streams</groupId> - <artifactId>streams-contrib</artifactId> - <version>0.1-SNAPSHOT</version> - </parent> - - <properties> - <tika.version>1.5</tika.version> - </properties> - - <dependencies> - <dependency> - <groupId>org.apache.streams</groupId> - <artifactId>streams-config</artifactId> - </dependency> - <dependency> - <groupId>org.apache.streams</groupId> - <artifactId>streams-core</artifactId> - </dependency> - <dependency> - <groupId>org.apache.streams</groupId> - <artifactId>streams-pojo</artifactId> - </dependency> - <dependency> - <groupId>org.apache.streams</groupId> - <artifactId>streams-processor-urls</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - </dependency> - <dependency> - <groupId>org.jsonschema2pojo</groupId> - <artifactId>jsonschema2pojo-core</artifactId> - </dependency> - <dependency> - <groupId>org.apache.tika</groupId> - <artifactId>tika-core</artifactId> - <version>${tika.version}</version> - </dependency> - <dependency> - <groupId>org.apache.tika</groupId> - <artifactId>tika-parsers</artifactId> - <version>${tika.version}</version> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> - <groupId>org.apache.streams</groupId> - <artifactId>streams-testing</artifactId> - <version>${project.version}</version> - <scope>test</scope> - <type>test-jar</type> - </dependency> - </dependencies> - - <build> - <sourceDirectory>src/main/java</sourceDirectory> - <testSourceDirectory>src/test/java</testSourceDirectory> - <resources> - <resource> - <directory>src/main/resources</directory> - </resource> - </resources> - <testResources> - <testResource> - <directory>src/test/resources</directory> - </testResource> - </testResources> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <executions> - <execution> - <id>add-source</id> - <phase>generate-sources</phase> - <goals> - <goal>add-source</goal> - </goals> - <configuration> - <sources> - <source>target/generated-sources/jsonschema2pojo/**/*.java</source> - </sources> - </configuration> - </execution> - <execution> - <id>add-source-jaxb2</id> - <phase>generate-sources</phase> - <goals> - <goal>add-source</goal> - </goals> - <configuration> - <sources> - <source>target/generated-sources/jaxb2</source> - </sources> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.jsonschema2pojo</groupId> - <artifactId>jsonschema2pojo-maven-plugin</artifactId> - <configuration> - <addCompileSourceRoot>true</addCompileSourceRoot> - <generateBuilders>true</generateBuilders> - <sourcePaths> - <sourcePath>${project.basedir}/src/main/jsonschema/org/apache/streams/tika/LinkDetails.json</sourcePath> - </sourcePaths> - <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory> - <targetPackage>org.apache.streams.tika</targetPackage> - <useLongIntegers>true</useLongIntegers> - <useJodaDates>true</useJodaDates> - </configuration> - <executions> - <execution> - <goals> - <goal>generate</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-contrib/streams-processor-tika/src/main/java/org/apache/streams/tika/CategoryParser.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-processor-tika/src/main/java/org/apache/streams/tika/CategoryParser.java b/streams-contrib/streams-processor-tika/src/main/java/org/apache/streams/tika/CategoryParser.java deleted file mode 100644 index 18ff747..0000000 --- a/streams-contrib/streams-processor-tika/src/main/java/org/apache/streams/tika/CategoryParser.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.streams.tika; - -import java.io.Serializable; -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.util.List; - -public class CategoryParser implements Serializable -{ - /** - * This method takes a URL and from that text alone determines what categories that URL belongs in. - * @param url - String URL to categorize - * @return categories - A List<String&rt; of categories the URL seemingly belongs in - */ - public static List<String> getCategoriesFromUrl(String url) { - - // Clean the URL to remove useless bits and encoding artifacts - String normalizedUrl = normalizeURL(url); - - // Break the url apart and get the good stuff - String[] keywords = tokenizeURL(normalizedUrl); - - return null; - } - - /** - * Removes the protocol, if it exists, from the front and - * removes any random encoding characters - * Extend this to do other url cleaning/pre-processing - * @param url - The String URL to normalize - * @return normalizedUrl - The String URL that has no junk or surprises - */ - private static String normalizeURL(String url) - { - // Decode URL to remove any %20 type stuff - String normalizedUrl = url; - try { - // I've used a URLDecoder that's part of Java here, - // but this functionality exists in most modern languages - // and is universally called url decoding - normalizedUrl = URLDecoder.decode(url, "UTF-8"); - } - catch(UnsupportedEncodingException uee) - { - System.err.println("Unable to Decode URL. Decoding skipped."); - uee.printStackTrace(); - } - - // Remove the protocol, http:// ftp:// or similar from the front - if (normalizedUrl.contains("://")) - normalizedUrl = normalizedUrl.split(":\\/\\/")[1]; - - // Room here to do more pre-processing - - return normalizedUrl; - } - - /** - * Takes apart the url into the pieces that make at least some sense - * This doesn't guarantee that each token is a potentially valid keyword, however - * because that would require actually iterating over them again, which might be - * seen as a waste. - * @param url - Url to be tokenized - * @return tokens - A String array of all the tokens - */ - private static String[] tokenizeURL(String url) - { - // I assume that we're going to use the whole URL to find tokens in - // If you want to just look in the GET parameters, or you want to ignore the domain - // or you want to use the domain as a token itself, that would have to be - // processed above the next line, and only the remaining parts split - String[] tokens = url.split("\\b|_"); - - // One could alternatively use a more complex regex to remove more invalid matches - // but this is subject to your (?:in)?ability to actually write the regex you want - - // These next two get rid of tokens that are too short, also. - - // Destroys anything that's not alphanumeric and things that are - // alphanumeric but only 1 character long - //String[] tokens = url.split("(?:[\\W_]+\\w)*[\\W_]+"); - - // Destroys anything that's not alphanumeric and things that are - // alphanumeric but only 1 or 2 characters long - //String[] tokens = url.split("(?:[\\W_]+\\w{1,2})*[\\W_]+"); - - return tokens; - } - - // How this would be used - public static void main(String[] args) - { - List<String> soQuestionUrlClassifications = getCategoriesFromUrl("http://stackoverflow.com/questions/10046178/pattern-matching-for-url-classification";); - List<String> googleQueryURLClassifications = getCategoriesFromUrl("https://www.google.com/search?sugexp=chrome,mod=18&sourceid=chrome&ie=UTF-8&q=spring+is+a+new+service+instance+created#hl=en&sugexp=ciatsh&gs_nf=1&gs_mss=spring%20is%20a%20new%20bean%20instance%20created&tok=lnAt2g0iy8CWkY65Te75sg&pq=spring%20is%20a%20new%20bean%20instance%20created&cp=6&gs_id=1l&xhr=t&q=urlencode&pf=p&safe=off&sclient=psy-ab&oq=url+en&gs_l=&pbx=1&bav=on.2,or.r_gc.r_pw.r_cp.r_qf.,cf.osb&fp=2176d1af1be1f17d&biw=1680&bih=965";); - } -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-contrib/streams-processor-tika/src/main/java/org/apache/streams/tika/TikaProcessor.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-processor-tika/src/main/java/org/apache/streams/tika/TikaProcessor.java b/streams-contrib/streams-processor-tika/src/main/java/org/apache/streams/tika/TikaProcessor.java deleted file mode 100644 index 6d66165..0000000 --- a/streams-contrib/streams-processor-tika/src/main/java/org/apache/streams/tika/TikaProcessor.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.streams.tika; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsonorg.JsonOrgModule; -import com.google.common.collect.Lists; -import org.apache.commons.lang.NotImplementedException; -import org.apache.streams.core.StreamsDatum; -import org.apache.streams.core.StreamsProcessor; -import org.apache.streams.jackson.StreamsJacksonMapper; -import org.apache.streams.pojo.json.Activity; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; - -/** - * References: - * Some helpful references to help - * Purpose URL - * ------------- ---------------------------------------------------------------- - * [Status Codes] http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html - * [Test Cases] http://greenbytes.de/tech/tc/httpredirects/ - * [t.co behavior] https://dev.twitter.com/docs/tco-redirection-behavior - */ - -public class TikaProcessor implements StreamsProcessor -{ - private final static String STREAMS_ID = "LinkCrawlerProcessor"; - - private final static Logger LOGGER = LoggerFactory.getLogger(TikaProcessor.class); - - private ObjectMapper mapper; - - @Override - public List<StreamsDatum> process(StreamsDatum entry) { - - List<StreamsDatum> result = Lists.newArrayList(); - - LOGGER.debug("{} processing {}", STREAMS_ID, entry.getDocument().getClass()); - - Activity activity; - - // get list of shared urls - if( entry.getDocument() instanceof Activity) { - - activity = (Activity) entry.getDocument(); - - } - else if(entry.getDocument() instanceof String) { - - try { - activity = mapper.readValue((String) entry.getDocument(), Activity.class); - } catch (Exception e) { - e.printStackTrace(); - LOGGER.warn(e.getMessage()); - return(Lists.newArrayList(entry)); - } - - } - else throw new NotImplementedException(); - - List<String> outputLinks = activity.getLinks(); - // for each - for( String link : outputLinks ) { - - LOGGER.debug( "pulling {}", link); - - try { - StreamsDatum outputDatum = expandLink(link, entry); - if( outputDatum != null ) - result.add(outputDatum); - } catch (Exception e) { - //drop unexpandable links - LOGGER.debug("Failed to expand link : {}", link); - LOGGER.debug("Excpetion expanding link : {}", e); - } - - } - - return result; - } - - private StreamsDatum expandLink(String link, StreamsDatum input) { - - LinkCrawler expander = new LinkCrawler((String)link); - expander.run(); - StreamsDatum datum = null; - if(input.getId() == null) - try { - datum = new StreamsDatum(this.mapper.writeValueAsString(expander.getArticle()), expander.getFinalURL()); - } catch (JsonProcessingException e) { - e.printStackTrace(); - return null; - } - else - try { - datum = new StreamsDatum(this.mapper.writeValueAsString(expander.getArticle()), input.getId()); - } catch (JsonProcessingException e) { - e.printStackTrace(); - return null; - } - datum.setSequenceid(input.getSequenceid()); - datum.setMetadata(input.getMetadata()); - datum.setTimestamp(input.getTimestamp()); - return datum; - - } - - @Override - public void prepare(Object o) { - this.mapper = StreamsJacksonMapper.getInstance(); - this.mapper.registerModule(new JsonOrgModule()); - } - - @Override - public void cleanUp() { - - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-contrib/streams-processor-tika/src/main/jsonschema/org/apache/streams/tika/BoilerPipeArticle.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-processor-tika/src/main/jsonschema/org/apache/streams/tika/BoilerPipeArticle.json b/streams-contrib/streams-processor-tika/src/main/jsonschema/org/apache/streams/tika/BoilerPipeArticle.json deleted file mode 100644 index 58f817e..0000000 --- a/streams-contrib/streams-processor-tika/src/main/jsonschema/org/apache/streams/tika/BoilerPipeArticle.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "type": "object", - "properties": { - "siteStatus" : { - "type" : "string", - "enum" : ["SUCCESS", "ERROR"] - }, - "title": { - "type": "string" - }, - "description": { - "type": "string" - }, - "body": { - "type": "string" - }, - "plainText": { - "type": "string" - }, - "medium": { - "type": "string" - }, - "author": { - "type": "string" - }, - "locale": { - "type": "string" - }, - "publishedDate": { - "type": "string", - "format" : "date-time" - }, - "lastModifiedDate": { - "type": "string", - "format" : "date-time" - }, - "imageURL": { - "type": "string" - }, - "languageDetected": { - "type": "object", - "properties": { - "languageCode": { - "type": "string" - }, - "isLanguageReasonablyCertain": { - "type": "boolean" - } - } - }, - "keywords": { - "type": "array", - "uniqueItems": true, - "items": { - "type": "string" - } - }, - "twitterCreator": { - "type": "string" - }, - "twitterSite": { - "type": "string" - }, - "facebookPage": { - "type": "string" - }, - "facebookApp": { - "type": "string" - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-contrib/streams-processor-tika/src/test/java/org/apache/streams/tika/TestCategoryParser.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-processor-tika/src/test/java/org/apache/streams/tika/TestCategoryParser.java b/streams-contrib/streams-processor-tika/src/test/java/org/apache/streams/tika/TestCategoryParser.java deleted file mode 100644 index f89006a..0000000 --- a/streams-contrib/streams-processor-tika/src/test/java/org/apache/streams/tika/TestCategoryParser.java +++ /dev/null @@ -1,30 +0,0 @@ -/*Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License.*/ - -package org.apache.streams.tika; - -import org.apache.commons.lang3.SerializationUtils; -import org.junit.Test; - -public class TestCategoryParser { - - @Test - public void testSerializability() { - CategoryParser parser = new CategoryParser(); - CategoryParser clone = SerializationUtils.clone(parser); - } -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-contrib/streams-processor-tika/src/test/java/org/apache/streams/tika/TestTikaProcessor.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-processor-tika/src/test/java/org/apache/streams/tika/TestTikaProcessor.java b/streams-contrib/streams-processor-tika/src/test/java/org/apache/streams/tika/TestTikaProcessor.java deleted file mode 100644 index 35f2d49..0000000 --- a/streams-contrib/streams-processor-tika/src/test/java/org/apache/streams/tika/TestTikaProcessor.java +++ /dev/null @@ -1,29 +0,0 @@ -/*Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License.*/ - -package org.apache.streams.tika; - -import org.apache.commons.lang3.SerializationUtils; -import org.junit.Test; - -public class TestTikaProcessor { - @Test - public void testSerializability() { - TikaProcessor processor = new TikaProcessor(); - TikaProcessor clone = SerializationUtils.clone(processor); - } -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-osgi-components/activity-consumer/osgi.bnd ---------------------------------------------------------------------- diff --git a/streams-osgi-components/activity-consumer/osgi.bnd b/streams-osgi-components/activity-consumer/osgi.bnd deleted file mode 100644 index 53f60f9..0000000 --- a/streams-osgi-components/activity-consumer/osgi.bnd +++ /dev/null @@ -1,9 +0,0 @@ -#-------------------------------------------------------------------------------- -# Licensed under Apache License 2.0 - http://www.apache.org/licenses/LICENSE-2.0 -#-------------------------------------------------------------------------------- - -#----------------------------------------------------------------- -# Use this file to add customized Bnd instructions for the bundle -#----------------------------------------------------------------- - - http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-osgi-components/activity-consumer/pom.xml ---------------------------------------------------------------------- diff --git a/streams-osgi-components/activity-consumer/pom.xml b/streams-osgi-components/activity-consumer/pom.xml deleted file mode 100644 index 72d4e71..0000000 --- a/streams-osgi-components/activity-consumer/pom.xml +++ /dev/null @@ -1,171 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>org.apache.streams.osgi.components</groupId> - <artifactId>streams-osgi-components</artifactId> - <version>0.4-incubating-SNAPSHOT</version> - </parent> - - <artifactId>activity-consumer</artifactId> - <name>${bundle.symbolicName} [${bundle.namespace}]</name> - - <packaging>bundle</packaging> - - <properties> - <bundle.symbolicName>activity-consumer-bundle</bundle.symbolicName> - <bundle.namespace>org.apache.streams.osgi.components.activityconsumer</bundle.namespace> - <commons.log>1.1</commons.log><!-- - <jackson.version>1.9.11</jackson.version>--> - <jackson.old.version>1.9.11</jackson.old.version> - </properties> - - <build> - <resources> - <resource> - <directory>src/main/resources</directory> - </resource> - <!-- - | example additional resource entries, useful when building Eclipse RCP applications - --> - <resource> - <directory>.</directory> - <includes> - <include>plugin.xml</include> - <include>plugin.properties</include> - <include>icons/**</include> - </includes> - </resource> - </resources> - <plugins> - - <plugin> - <groupId>org.ops4j</groupId> - <artifactId>maven-pax-plugin</artifactId> - <!-- - | enable improved OSGi compilation support for the bundle life-cycle. - | to switch back to the standard bundle life-cycle, move this setting - | down to the maven-bundle-plugin section - --> - <extensions>true</extensions> - </plugin> - <plugin> - <groupId>org.apache.felix</groupId> - <artifactId>maven-bundle-plugin</artifactId> - <version>1.4.3</version> - <!-- - | the following instructions build a simple set of public/private classes into an OSGi bundle - --> - <configuration> - <instructions> - <Bundle-SymbolicName>${bundle.symbolicName}</Bundle-SymbolicName> - <Bundle-Version>${project.version}</Bundle-Version> - <Export-Package>${bundle.namespace};version="${project.version}",org.apache.streams.osgi.components.activityconsumer.impl</Export-Package> - <Private-Package>${bundle.namespace}.impl.*</Private-Package> - <Import-Package>org.apache.streams.osgi.components.activityconsumer,org.apache.streams.osgi.components.activityconsumer.impl,org.apache.commons.logging,org.codehaus.jackson.*;version="${jackson.version}",javax.xml.datatype, javax.xml.namespace, javax.xml.parsers, org.joda.time, org.joda.time.format, org.w3c.dom, org.w3c.dom.bootstrap, org.w3c.dom.ls, org.xml.sax, org.apache.rave.model, org.apache.rave.portal.model.impl, org.apache.streams.cassandra.model - </Import-Package> - </instructions> - </configuration> - </plugin> - </plugins> - </build> - - <dependencies> - <dependency> - <groupId>org.osgi</groupId> - <artifactId>osgi_R4_core</artifactId> - <version>1.0</version> - <scope>provided</scope> - <optional>true</optional> - </dependency> - - <dependency> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-mrbean</artifactId> - <version>${jackson.old.version}</version> - </dependency> - - <dependency> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-mapper-asl</artifactId> - <version>${jackson.old.version}</version> - </dependency> - - <dependency> - <groupId>org.osgi</groupId> - <artifactId>osgi_R4_compendium</artifactId> - <version>1.0</version> - <scope>provided</scope> - <optional>true</optional> - </dependency> - <dependency> - <groupId>commons-logging</groupId> - <artifactId>commons-logging-api</artifactId> - <version>${commons.log}</version> - <scope>provided</scope> - </dependency> - - <dependency> - <groupId>org.apache.rave</groupId> - <artifactId>rave-core</artifactId> - <version>${rave.version}</version> - <exclusions> - <exclusion> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>org.apache.rave</groupId> - <artifactId>rave-core-api</artifactId> - <version>${rave.version}</version> - <exclusions> - <exclusion> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>jcl-over-slf4j</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.streams</groupId> - <artifactId>streams-persist-cassandra</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.streams</groupId> - <artifactId>streams-testing</artifactId> - <version>${project.version}</version> - <scope>test</scope> - <type>test-jar</type> - </dependency> - </dependencies> - -</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-osgi-components/activity-consumer/src/main/java/org/apache/streams/osgi/components/activityconsumer/ActivityConsumer.java ---------------------------------------------------------------------- diff --git a/streams-osgi-components/activity-consumer/src/main/java/org/apache/streams/osgi/components/activityconsumer/ActivityConsumer.java b/streams-osgi-components/activity-consumer/src/main/java/org/apache/streams/osgi/components/activityconsumer/ActivityConsumer.java deleted file mode 100644 index 2fd6bd3..0000000 --- a/streams-osgi-components/activity-consumer/src/main/java/org/apache/streams/osgi/components/activityconsumer/ActivityConsumer.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.streams.osgi.components.activityconsumer; - - -import org.codehaus.jackson.annotate.JsonTypeInfo; - -import java.net.URI; - - -@JsonTypeInfo(use= JsonTypeInfo.Id.CLASS, include=JsonTypeInfo.As.PROPERTY, property="@class") -public interface ActivityConsumer { - public String receive(String activity); - public void init(); - public URI getSrc(); - public void setSrc(String src); - public void setInRoute(String route); - public String getInRoute(); - public String getAuthToken(); - public void setAuthToken(String token); - public boolean isAuthenticated(); - public void setAuthenticated(boolean authenticated); -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-osgi-components/activity-consumer/src/main/java/org/apache/streams/osgi/components/activityconsumer/ActivityConsumerWarehouse.java ---------------------------------------------------------------------- diff --git a/streams-osgi-components/activity-consumer/src/main/java/org/apache/streams/osgi/components/activityconsumer/ActivityConsumerWarehouse.java b/streams-osgi-components/activity-consumer/src/main/java/org/apache/streams/osgi/components/activityconsumer/ActivityConsumerWarehouse.java deleted file mode 100644 index 9153d26..0000000 --- a/streams-osgi-components/activity-consumer/src/main/java/org/apache/streams/osgi/components/activityconsumer/ActivityConsumerWarehouse.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.streams.osgi.components.activityconsumer; - -/** - * Public API representing an example OSGi service - */ -public interface ActivityConsumerWarehouse { - - public void register(ActivityConsumer activityConsumer); - public ActivityConsumer findConsumerBySrc(String src); - public int getConsumersCount(); - -} - http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-osgi-components/activity-consumer/src/main/java/org/apache/streams/osgi/components/activityconsumer/impl/ActivityConsumerWarehouseImpl.java ---------------------------------------------------------------------- diff --git a/streams-osgi-components/activity-consumer/src/main/java/org/apache/streams/osgi/components/activityconsumer/impl/ActivityConsumerWarehouseImpl.java b/streams-osgi-components/activity-consumer/src/main/java/org/apache/streams/osgi/components/activityconsumer/impl/ActivityConsumerWarehouseImpl.java deleted file mode 100644 index 5e40ba5..0000000 --- a/streams-osgi-components/activity-consumer/src/main/java/org/apache/streams/osgi/components/activityconsumer/impl/ActivityConsumerWarehouseImpl.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.streams.osgi.components.activityconsumer.impl; - -import java.util.Date; -import java.util.HashMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.streams.osgi.components.activityconsumer.ActivityConsumerWarehouse; -import org.apache.streams.osgi.components.activityconsumer.ActivityConsumer; - -public class ActivityConsumerWarehouseImpl implements ActivityConsumerWarehouse { - private static final transient Log LOG = LogFactory.getLog(ActivityConsumerWarehouseImpl.class); - - private HashMap<String,ActivityConsumer> consumers; - - public ActivityConsumerWarehouseImpl(){ - consumers = new HashMap<String, ActivityConsumer>(); - } - - public void register(ActivityConsumer activityConsumer) { - - //key in warehouse is the activity publisher URI source - consumers.put(activityConsumer.getSrc().toASCIIString(), activityConsumer); - activityConsumer.init(); - - - } - - public ActivityConsumer findConsumerBySrc(String src){ - return consumers.get(src); - } - - - public int getConsumersCount(){ - return consumers.size(); - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ce7aa24d/streams-osgi-components/activity-consumer/src/main/java/org/apache/streams/osgi/components/activityconsumer/impl/PushActivityConsumer.java ---------------------------------------------------------------------- diff --git a/streams-osgi-components/activity-consumer/src/main/java/org/apache/streams/osgi/components/activityconsumer/impl/PushActivityConsumer.java b/streams-osgi-components/activity-consumer/src/main/java/org/apache/streams/osgi/components/activityconsumer/impl/PushActivityConsumer.java deleted file mode 100644 index 1d4a7c1..0000000 --- a/streams-osgi-components/activity-consumer/src/main/java/org/apache/streams/osgi/components/activityconsumer/impl/PushActivityConsumer.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.streams.osgi.components.activityconsumer.impl; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.streams.cassandra.model.CassandraActivityStreamsEntry; -import org.apache.streams.osgi.components.activityconsumer.ActivityConsumer; -import org.codehaus.jackson.map.DeserializationConfig; -import org.codehaus.jackson.map.ObjectMapper; - -import java.net.URI; -import java.net.URISyntaxException; -import java.util.List; -import java.util.ArrayList; - -public class PushActivityConsumer implements ActivityConsumer { - - private static final transient Log LOG = LogFactory.getLog(PushActivityConsumer.class); - - private URI src; - - - private String authToken; - - private boolean authenticated; - - private String inRoute; - - public PushActivityConsumer(){ - } - - - public URI getSrc() { - return src; - } - - public void setSrc(String src) { - try{ - this.src = new URI(src); - - } catch (URISyntaxException e) { - this.src=null; - } - } - - public String getAuthToken() { - return authToken; - } - - public void setAuthToken(String authToken) { - this.authToken = authToken; - } - - public boolean isAuthenticated() { - return authenticated; - } - - public void setAuthenticated(boolean authenticated) { - this.authenticated = authenticated; - } - - public String receive (String activity){ - //receive activities...do anything that is necessary - LOG.info("a message I published: " + activity); - return activity; - //pass off to activity splitter - - } - - public List<String> split(String activities) { - LOG.info("I am going to split this message: " + activities); - - List<String> activitiesList = new ArrayList<String>(); - activitiesList.add(activities); - return activitiesList; - } - - public void init(){ - //any initialization... - } - - public String getInRoute() { - return inRoute; - } - - public void setInRoute(String inRoute) { - this.inRoute = inRoute; - } -}
![http://abs.twimg.com/images/themes/theme1/bg.png\",\"profile_background_image_url_](http://abs.twimg.com/images/themes/theme1/bg.png\",\"profile_background_image_url_){kind=link}
![https://abs.twimg.com/images/themes/theme1/bg.png\",\"profile_background_tile\":false,\"profile_image_url\":\"http://pbs.twimg.com/profile_images/613854495/winchester_sociallogo_normal.jpg\",\"profile_image_url_https\":\"https://pbs.twimg.com/profile_images/613854495/winchester_sociallogo_normal.jpg\",\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"symbols\":[],\"urls\":[{\"url\":\"http://t.co/SH5YZGpdRx\",\"expanded_url\":\"http://ow.ly/2C2XL1\",\"display_url\":\"ow.ly/2C2XL1\",\"indices\":[118,140]}],\"user_mentions\":[]},\"favorited\":false,\"retweeted\":false](https://abs.twimg.com/images/themes/theme1/bg.png\",\"profile_background_tile\":false,\"profile_image_url\":\"http://pbs.twimg.com/profile_images/613854495/winchester_sociallogo_normal.jpg\",\"profile_image_url_https\":\"https://pbs.twimg.com/profile_images/613854495/winchester_sociallogo_normal.jpg\",\"profile_link_color\":\"0084B4\",\"profile_sidebar_border_color\":\"C0DEED\",\"profile_sidebar_fill_color\":\"DDEEF6\",\"profile_text_color\":\"333333\",\"profile_use_background_image\":true,\"default_profile\":true,\"default_profile_image\":false,\"following\":null,\"follow_request_sent\":null,\"notifications\":null},\"geo\":null,\"coordinates\":null,\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"entities\":{\"hashtags\":[],\"symbols\":[],\"urls\":[{\"url\":\"http://t.co/SH5YZGpdRx\",\"expanded_url\":\"http://ow.ly/2C2XL1\",\"display_url\":\"ow.ly/2C2XL1\",\"indices\":[118,140]}],\"user_mentions\":[]},\"favorited\":false,\"retweeted\":false){kind=link}
![https://si0.twimg.com/sticky/default_profile_images/default_profile_1_normal.png\"},\"displayName\":\"Sabine](https://si0.twimg.com/sticky/default_profile_images/default_profile_1_normal.png\"},\"displayName\":\"Sabine){kind=link}
![http://www.wcgworld.com/assets/img/home_banner.png\",\"og:url\":\"http://www.wcgworld.com/\"},\"normalized](http://www.wcgworld.com/assets/img/home_banner.png\",\"og:url\":\"http://www.wcgworld.com/\"},\"normalized){kind=link}
![http://www.wcgworld.com/assets/img/home_banner.png\",\"plainText\":null}\n"](http://www.wcgworld.com/assets/img/home_banner.png\",\"plainText\":null}\n"){kind=link}