http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java deleted file mode 100644 index fbd5e52..0000000 --- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/DatasiftStreamProvider.java +++ /dev/null @@ -1,242 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ - -package org.apache.streams.datasift.provider; - -import com.datasift.client.DataSiftClient; -import com.datasift.client.DataSiftConfig; -import com.datasift.client.core.Stream; -import com.datasift.client.stream.DeletedInteraction; -import com.datasift.client.stream.Interaction; -import com.datasift.client.stream.StreamEventListener; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; -import com.google.common.collect.Queues; -import com.typesafe.config.Config; -import org.apache.streams.config.StreamsConfigurator; -import org.apache.streams.core.StreamsDatum; -import org.apache.streams.core.StreamsProvider; -import org.apache.streams.core.StreamsResultSet; -import org.apache.streams.datasift.DatasiftConfiguration; -import org.apache.streams.datasift.util.StreamsDatasiftMapper; -import org.joda.time.DateTime; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.math.BigInteger; -import java.util.Map; -import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedQueue; - -/** - * Requires Java Version 1.7! - * {@code DatasiftStreamProvider} is an implementation of the {@link org.apache.streams.core.StreamsProvider} interface. The provider - * uses the Datasift java api to make connections. A single provider creates one connection per StreamHash in the configuration. - */ -public class DatasiftStreamProvider implements StreamsProvider { - - private final static String STREAMS_ID = "DatasiftStreamProvider"; - - private final static Logger LOGGER = LoggerFactory.getLogger(DatasiftStreamProvider.class); - - private DatasiftConfiguration config; - private ConcurrentLinkedQueue<Interaction> interactions = new ConcurrentLinkedQueue<Interaction>(); - private Map<String, DataSiftClient> clients; - private StreamEventListener eventListener; - private ObjectMapper mapper; - - /** - * Constructor that searches for available configurations - * - * @param listener {@link com.datasift.client.stream.StreamEventListener} that handles deletion notices received from twitter. - */ - public DatasiftStreamProvider(StreamEventListener listener) { - this(listener, null); - } - - // to set up a webhook we need to be able to return a reference to this queue - public Queue<Interaction> getInteractions() { - return interactions; - } - - /** - * @param listener {@link com.datasift.client.stream.StreamEventListener} that handles deletion notices received from twitter. - * @param config Configuration to use - */ - public DatasiftStreamProvider(StreamEventListener listener, DatasiftConfiguration config) { - if (config == null) { - Config datasiftConfig = StreamsConfigurator.config.getConfig("datasift"); - this.config = DatasiftStreamConfigurator.detectConfiguration(datasiftConfig); - } else { - this.config = config; - } - this.eventListener = listener; - } - - @Override - public String getId() { - return STREAMS_ID; - } - - @Override - public void startStream() { - - Preconditions.checkNotNull(this.config); - Preconditions.checkNotNull(this.config.getStreamHash()); - Preconditions.checkNotNull(this.config.getStreamHash().get(0)); - Preconditions.checkNotNull(this.config.getApiKey()); - Preconditions.checkNotNull(this.config.getUserName()); - Preconditions.checkNotNull(this.clients); - - for (String hash : this.config.getStreamHash()) { - startStreamForHash(hash); - } - - } - - /** - * Creates a connection to datasift and starts collection of data from the resulting string. - * - * @param streamHash - */ - public void startStreamForHash(String streamHash) { - shutDownStream(streamHash); - DataSiftClient client = getNewClient(this.config.getUserName(), this.config.getApiKey()); - client.liveStream().onStreamEvent(this.eventListener); - client.liveStream().onError(new ErrorHandler(this, streamHash)); - - client.liveStream().subscribe(new Subscription(Stream.fromString(streamHash), this.interactions)); - synchronized (this.clients) { - this.clients.put(streamHash, client); - } - } - - /** - * Exposed for testing purposes. - * - * @param userName - * @param apiKey - * @return - */ - protected DataSiftClient getNewClient(String userName, String apiKey) { - return new DataSiftClient(new DataSiftConfig(userName, apiKey)); - } - - - /** - * If a stream has been opened for the supplied stream hash, that stream will be shutdown. - * - * @param streamHash - */ - public void shutDownStream(String streamHash) { - synchronized (clients) { - if (!this.clients.containsKey(streamHash)) - return; - DataSiftClient client = this.clients.get(streamHash); - LOGGER.debug("Shutting down stream for hash: {}", streamHash); - client.shutdown(); - this.clients.remove(client); - } - } - - /** - * Shuts down all open streams from datasift. - */ - public void stop() { - synchronized (clients) { - for (DataSiftClient client : this.clients.values()) { - client.shutdown(); - } - } - } - - // PRIME EXAMPLE OF WHY WE NEED NEW INTERFACES FOR PROVIDERS - @Override - //This is a hack. It is only like this because of how perpetual streams work at the moment. Read list server to debate/vote for new interfaces. - public StreamsResultSet readCurrent() { - Queue<StreamsDatum> datums = Queues.newConcurrentLinkedQueue(); - StreamsDatum datum = null; - Interaction interaction; - while (!this.interactions.isEmpty()) { - interaction = this.interactions.poll(); - try { - datum = new StreamsDatum(this.mapper.writeValueAsString(interaction.getData()), interaction.getData().get("interaction").get("id").textValue()); - } catch (JsonProcessingException jpe) { - LOGGER.error("Exception while converting Interaction to String : {}", jpe); - } - if (datum != null) { - while (!datums.offer(datum)) { - Thread.yield(); - } - } - - } - return new StreamsResultSet(datums); - } - - @Override - public StreamsResultSet readNew(BigInteger sequence) { - return null; - } - - public StreamsResultSet readRange(DateTime start, DateTime end) { - return null; - } - - @Override - public boolean isRunning() { - return this.clients != null && this.clients.size() > 0; - } - - @Override - public void prepare(Object configurationObject) { - this.interactions = new ConcurrentLinkedQueue<Interaction>(); - this.clients = Maps.newHashMap(); - this.mapper = StreamsDatasiftMapper.getInstance(); - } - - @Override - public void cleanUp() { - stop(); - } - - public DatasiftConfiguration getConfig() { - return config; - } - - public void setConfig(DatasiftConfiguration config) { - this.config = config; - } - - - /** - * THIS CLASS NEEDS TO BE REPLACED/OVERRIDDEN BY ALL USERS. TWITTERS TERMS OF SERVICE SAYS THAT EVERYONE MUST - * DELETE TWEETS FROM THEIR DATA STORE IF THEY RECEIVE A DELETE NOTICE. - */ - public static class DeleteHandler extends StreamEventListener { - - public void onDelete(DeletedInteraction di) { - //go off and delete the interaction if you have it stored. This is a strict requirement! - LOGGER.info("DELETED:\n " + di); - } - } - -}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/ErrorHandler.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/ErrorHandler.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/ErrorHandler.java deleted file mode 100644 index d5f19d5..0000000 --- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/ErrorHandler.java +++ /dev/null @@ -1,47 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ - -package org.apache.streams.datasift.provider; - -import com.datasift.client.stream.ErrorListener; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Listens for exceptions from the datasift streams and resets connections on errors. - */ -public class ErrorHandler extends ErrorListener { - - private static final Logger LOGGER = LoggerFactory.getLogger(ErrorHandler.class); - - private String streamHash; - private DatasiftStreamProvider provider; - - public ErrorHandler(DatasiftStreamProvider provider, String streamHash) { - this.provider = provider; - this.streamHash = streamHash; - } - - @Override - public void exceptionCaught(Throwable throwable) { - LOGGER.error("DatasiftClient received Exception : {}", throwable); - LOGGER.info("Attempting to restart client for stream hash : {}", this.streamHash); - this.provider.startStreamForHash(this.streamHash); - } -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/Subscription.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/Subscription.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/Subscription.java deleted file mode 100644 index 058ba1d..0000000 --- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/provider/Subscription.java +++ /dev/null @@ -1,60 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ - -package org.apache.streams.datasift.provider; - -import com.datasift.client.core.Stream; -import com.datasift.client.stream.DataSiftMessage; -import com.datasift.client.stream.Interaction; -import com.datasift.client.stream.StreamSubscription; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Queue; - -/** - * Adds incomming {@link com.datasift.client.stream.Interaction} to the queue for the provider. - */ -public class Subscription extends StreamSubscription { - - private static final Logger LOGGER = LoggerFactory.getLogger(Subscription.class); - private Queue<Interaction> sharedQueue; - - public Subscription(Stream stream, Queue<Interaction> sharedQueue) { - super(stream); - this.sharedQueue = sharedQueue; - } - - @Override - public void onDataSiftLogMessage(DataSiftMessage dataSiftMessage) { - if (dataSiftMessage.isError()) //should we restart the subscription on error? - LOGGER.error("Datasift Error : {}", dataSiftMessage.getMessage()); - else if (dataSiftMessage.isWarning()) - LOGGER.warn("Datasift Warning : {}", dataSiftMessage.getMessage()); - else - LOGGER.info("Datasift Info : {}", dataSiftMessage.getMessage()); - } - - @Override - public void onMessage(Interaction interaction) { - while (!this.sharedQueue.offer(interaction)) { - Thread.yield(); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java deleted file mode 100644 index b587cd6..0000000 --- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftActivitySerializer.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.streams.datasift.serializer; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.streams.data.ActivitySerializer; -import org.apache.streams.datasift.Datasift; -import org.apache.streams.datasift.util.StreamsDatasiftMapper; -import org.apache.streams.exceptions.ActivitySerializerException; -import org.apache.streams.pojo.json.Activity; - -import java.util.List; - -/** - * - */ -public class DatasiftActivitySerializer implements ActivitySerializer<Datasift> { - - private static final ObjectMapper MAPPER = StreamsDatasiftMapper.getInstance(); - - @Override - public String serializationFormat() { - return null; - } - - @Override - public Datasift serialize(Activity deserialized) throws ActivitySerializerException { - return null; - } - - @Override - public Activity deserialize(Datasift serialized) throws ActivitySerializerException { - ActivitySerializer serializer = DatasiftEventClassifier.bestSerializer(serialized); - return serializer.deserialize(serialized); - } - - public Activity deserialize(String json) throws ActivitySerializerException { - try { - return deserialize(MAPPER.readValue(json, Datasift.class)); - } catch (Exception e) { - throw new ActivitySerializerException(e); - } - } - - @Override - public List<Activity> deserializeAll(List<Datasift> serializedList) { - return null; - } -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java deleted file mode 100644 index 7d7d547..0000000 --- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftEventClassifier.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.streams.datasift.serializer; - -import org.apache.streams.data.ActivitySerializer; -import org.apache.streams.datasift.Datasift; -import org.apache.streams.datasift.instagram.Instagram; -import org.apache.streams.datasift.interaction.Interaction; -import org.apache.streams.datasift.twitter.Twitter; - -/** - * Created by sblackmon on 11/6/14. - */ -public class DatasiftEventClassifier { - - public static Class detectClass(Datasift event) { - - if(event.getTwitter() != null) { - return Twitter.class; - } else if(event.getInstagram() != null) { - return Instagram.class; - } else { - return Interaction.class; - } - } - - public static ActivitySerializer bestSerializer(Datasift event) { - - if(event.getTwitter() != null) { - return DatasiftTwitterActivitySerializer.getInstance(); - } else if(event.getInstagram() != null) { - return DatasiftInstagramActivitySerializer.getInstance(); - } else { - return DatasiftInteractionActivitySerializer.getInstance(); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java deleted file mode 100644 index 1ed805d..0000000 --- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializer.java +++ /dev/null @@ -1,125 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.streams.datasift.serializer; - -import com.google.common.base.Joiner; -import com.google.common.base.Optional; -import com.google.common.collect.Lists; -import org.apache.streams.datasift.Datasift; -import org.apache.streams.datasift.instagram.From; -import org.apache.streams.datasift.instagram.Instagram; -import org.apache.streams.instagram.serializer.util.InstagramActivityUtil; -import org.apache.streams.pojo.extensions.ExtensionUtil; -import org.apache.streams.pojo.json.Activity; -import org.apache.streams.pojo.json.Actor; -import org.apache.streams.pojo.json.Image; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.Map; - -import static org.apache.streams.data.util.ActivityUtil.ensureExtensions; - -/** - * - */ -public class DatasiftInstagramActivitySerializer extends DatasiftInteractionActivitySerializer { - - private static final Logger LOGGER = LoggerFactory.getLogger(DatasiftInstagramActivitySerializer.class); - - private static DatasiftInstagramActivitySerializer instance = new DatasiftInstagramActivitySerializer(); - - public static DatasiftInstagramActivitySerializer getInstance() { - return instance; - } - - @Override - public Activity convert(Datasift event) { - Activity activity = super.convert(event); - - Instagram instagram = event.getInstagram(); - - activity.setActor(buildActor(event, instagram)); - activity.setId(formatId(activity.getVerb(), event.getInteraction().getId())); - activity.setProvider(InstagramActivityUtil.getProvider()); - activity.setLinks(getLinks(event.getInstagram())); - - activity.setVerb(selectVerb(event)); - addInstagramExtensions(activity, instagram); - return activity; - } - - /** - * Gets links from the object - * @return - */ - private List<String> getLinks(Instagram instagram) { - List<String> result = Lists.newLinkedList(); - if( instagram.getMedia() != null ) { - result.add(instagram.getMedia().getImage()); - result.add(instagram.getMedia().getVideo()); - } - return result; - } - - public Actor buildActor(Datasift event, Instagram instagram) { - Actor actor = super.buildActor(event.getInteraction()); - From user = instagram.getFrom(); - - actor.setDisplayName(firstStringIfNotNull(user.getFullName())); - actor.setId(formatId(Optional.fromNullable( - user.getId()) - .or(actor.getId()))); - - Image profileImage = new Image(); - String profileUrl = null; - profileUrl = user.getProfilePicture(); - profileImage.setUrl(profileUrl); - actor.setImage(profileImage); - - return actor; - } - - public void addInstagramExtensions(Activity activity, Instagram instagram) { - Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity); - List<String> hashTags; - if(instagram.getMedia() != null) { - hashTags = instagram.getMedia().getTags(); - extensions.put("hashtags", hashTags); - extensions.put("keywords", activity.getContent()); - } else { - extensions.put("keywords", activity.getContent()); - - } - - } - - private String selectVerb(Datasift event) { - if( event.getInteraction().getSubtype().equals("like")) - return "like"; - else - return "post"; - } - - public static String formatId(String... idparts) { - return Joiner.on(":").join(Lists.asList("id:instagram", idparts)); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializer.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializer.java deleted file mode 100644 index 79bf940..0000000 --- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializer.java +++ /dev/null @@ -1,247 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.streams.datasift.serializer; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import org.apache.streams.data.ActivitySerializer; -import org.apache.streams.datasift.Datasift; -import org.apache.streams.datasift.interaction.Interaction; -import org.apache.streams.datasift.links.Links; -import org.apache.streams.datasift.util.StreamsDatasiftMapper; -import org.apache.streams.pojo.extensions.ExtensionUtil; -import org.apache.streams.pojo.json.Activity; -import org.apache.streams.pojo.json.ActivityObject; -import org.apache.streams.pojo.json.Actor; -import org.apache.streams.pojo.json.Generator; -import org.apache.streams.pojo.json.Icon; -import org.apache.streams.pojo.json.Image; -import org.apache.streams.pojo.json.Provider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.streams.data.util.ActivityUtil.ensureExtensions; - -/** - * - */ -public class DatasiftInteractionActivitySerializer implements ActivitySerializer<Datasift>, Serializable { - - private static final Logger LOGGER = LoggerFactory.getLogger(DatasiftInteractionActivitySerializer.class); - - private static DatasiftInteractionActivitySerializer instance = new DatasiftInteractionActivitySerializer(); - - public static DatasiftInteractionActivitySerializer getInstance() { - return instance; - } - - ObjectMapper mapper = StreamsDatasiftMapper.getInstance(); - - @Override - public String serializationFormat() { - return "application/json+datasift.com.v1.1"; - } - - @Override - public Datasift serialize(Activity deserialized) { - throw new UnsupportedOperationException("Cannot currently serialize to Datasift JSON"); - } - - public Activity deserialize(String datasiftJson) { - try { - return deserialize(this.mapper.readValue(datasiftJson, Datasift.class)); - } catch (Exception e) { - LOGGER.error("Exception while trying convert,\n {},\n to a Datasift object.", datasiftJson); - LOGGER.error("Exception : {}", e); - throw new RuntimeException(e); - } - } - - @Override - public Activity deserialize(Datasift serialized) { - - try { - - Activity activity = convert(serialized); - - return activity; - - } catch (Exception e) { - throw new IllegalArgumentException("Unable to deserialize", e); - } - - } - - @Override - public List<Activity> deserializeAll(List<Datasift> datasifts) { - List<Activity> activities = Lists.newArrayList(); - for( Datasift datasift : datasifts ) { - activities.add(deserialize(datasift)); - } - return activities; - } - - public static Generator buildGenerator(Interaction interaction) { - Generator generator = new Generator(); - generator.setDisplayName(interaction.getSource()); - generator.setId(interaction.getSource()); - return generator; - } - - public static Icon getIcon(Interaction interaction) { - return null; - } - - public static Provider buildProvider(Interaction interaction) { - Provider provider = new Provider(); - provider.setId("id:providers:"+interaction.getType()); - provider.setDisplayName(interaction.getType()); - return provider; - } - - public static String getUrls(Interaction interaction) { - return null; - } - - public static void addDatasiftExtension(Activity activity, Datasift datasift) { - Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity); - extensions.put("datasift", datasift); - } - - public static String formatId(String... idparts) { - return Joiner.on(":").join(Lists.asList("id:datasift", idparts)); - } - - public Activity convert(Datasift event) { - - Preconditions.checkNotNull(event); - Preconditions.checkNotNull(event.getInteraction()); - - Activity activity = new Activity(); - activity.setActor(buildActor(event.getInteraction())); - activity.setVerb(selectVerb(event)); - activity.setObject(buildActivityObject(event.getInteraction())); - activity.setId(formatId(activity.getVerb(), event.getInteraction().getId())); - activity.setTarget(buildTarget(event.getInteraction())); - activity.setPublished(event.getInteraction().getCreatedAt()); - activity.setGenerator(buildGenerator(event.getInteraction())); - activity.setIcon(getIcon(event.getInteraction())); - activity.setProvider(buildProvider(event.getInteraction())); - activity.setTitle(event.getInteraction().getTitle()); - activity.setContent(event.getInteraction().getContent()); - activity.setUrl(event.getInteraction().getLink()); - activity.setLinks(getLinks(event)); - addDatasiftExtension(activity, event); - if( event.getInteraction().getGeo() != null) { - addLocationExtension(activity, event.getInteraction()); - } - return activity; - } - - private String selectVerb(Datasift event) { - return "post"; - } - - public Actor buildActor(Interaction interaction) { - Actor actor = new Actor(); - org.apache.streams.datasift.interaction.Author author = interaction.getAuthor(); - if(author == null) { - LOGGER.warn("Interaction does not contain author information."); - return actor; - } - String userName = author.getUsername(); - String name = author.getName(); - Long id = author.getId(); - if(userName != null) { - actor.setDisplayName(userName); - } else { - actor.setDisplayName(name); - } - - if(id != null) { - actor.setId(id.toString()); - } else { - if(userName != null) - actor.setId(userName); - else - actor.setId(name); - } - Image image = new Image(); - image.setUrl(interaction.getAuthor().getAvatar()); - actor.setImage(image); - if (interaction.getAuthor().getLink()!=null){ - actor.setUrl(interaction.getAuthor().getLink()); - } - return actor; - } - - public static ActivityObject buildActivityObject(Interaction interaction) { - ActivityObject actObj = new ActivityObject(); - actObj.setObjectType(interaction.getContenttype()); - actObj.setUrl(interaction.getLink()); - actObj.setId(formatId("post",interaction.getId())); - actObj.setContent(interaction.getContent()); - - return actObj; - } - - public static List<String> getLinks(Datasift event) { - List<String> result = Lists.newArrayList(); - Links links = event.getLinks(); - if(links == null) - return null; - for(Object link : links.getNormalizedUrl()) { - if(link != null) { - if(link instanceof String) { - result.add((String) link); - } else { - LOGGER.warn("link is not of type String : {}", link.getClass().getName()); - } - } - } - return result; - } - - public static ActivityObject buildTarget(Interaction interaction) { - return null; - } - - public static void addLocationExtension(Activity activity, Interaction interaction) { - Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity); - Map<String, Object> location = new HashMap<String, Object>(); - Map<String, Double> coordinates = new HashMap<String, Double>(); - coordinates.put("latitude", interaction.getGeo().getLatitude()); - coordinates.put("longitude", interaction.getGeo().getLongitude()); - location.put("coordinates", coordinates); - extensions.put("location", location); - } - - public static String firstStringIfNotNull(List<Object> list) { - if( list != null && list.size() > 0) { - return (String) list.get(0); - } else return null; - } -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializer.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializer.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializer.java deleted file mode 100644 index cbd29dd..0000000 --- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializer.java +++ /dev/null @@ -1,271 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ -package org.apache.streams.datasift.serializer; - - -import com.google.common.base.Joiner; -import com.google.common.base.Optional; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import org.apache.streams.datasift.Datasift; -import org.apache.streams.datasift.interaction.Author; -import org.apache.streams.datasift.interaction.Interaction; -import org.apache.streams.datasift.twitter.DatasiftTwitterUser; -import org.apache.streams.datasift.twitter.Retweet; -import org.apache.streams.datasift.twitter.Twitter; -import org.apache.streams.pojo.extensions.ExtensionUtil; -import org.apache.streams.pojo.json.Activity; -import org.apache.streams.pojo.json.Actor; -import org.apache.streams.pojo.json.Image; -import org.apache.streams.twitter.converter.util.TwitterActivityUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * - */ -public class DatasiftTwitterActivitySerializer extends DatasiftInteractionActivitySerializer { - - private static final Logger LOGGER = LoggerFactory.getLogger(DatasiftTwitterActivitySerializer.class); - - private static DatasiftTwitterActivitySerializer instance = new DatasiftTwitterActivitySerializer(); - - public static DatasiftTwitterActivitySerializer getInstance() { - return instance; - } - - @Override - public Activity convert(Datasift event) { - Activity activity = new Activity(); - Twitter twitter = event.getTwitter(); - boolean retweet = twitter.getRetweet() != null; - - activity.setActor(buildActor(event, twitter)); //TODO - if(retweet) { - activity.setVerb("share"); - } else { - activity.setVerb("post"); - } - activity.setObject(buildActivityObject(event.getInteraction())); - activity.setId(formatId(activity.getVerb(), event.getInteraction().getId())); - activity.setTarget(buildTarget(event.getInteraction())); - activity.setPublished(event.getInteraction().getCreatedAt()); - activity.setGenerator(buildGenerator(event.getInteraction())); - activity.setIcon(getIcon(event.getInteraction())); - activity.setProvider(TwitterActivityUtil.getProvider()); - activity.setTitle(event.getInteraction().getTitle()); - activity.setContent(event.getInteraction().getContent()); - activity.setUrl(event.getInteraction().getLink()); - if(retweet) - activity.setLinks(getLinks(twitter.getRetweet())); - else - activity.setLinks(getLinks(twitter)); - addDatasiftExtension(activity, event); - if( twitter.getGeo() != null) { - addLocationExtension(activity, twitter); - } - addTwitterExtensions(activity, twitter, event.getInteraction()); - return activity; - } - - /** - * Get the links from this tweet as a list - * @param twitter - * @return the links from the tweet - */ - public List<String> getLinks(Twitter twitter) { - return getLinks(twitter.getLinks()); - } - - /** - * Get the links from this tweet as a list - * @param retweet - * @return the links from the tweet - */ - public List<String> getLinks(Retweet retweet) { - return getLinks(retweet.getLinks()); - } - - /** - * Converts the list of objects to a list of strings - * @param links - * @return - */ - private List<String> getLinks(List<Object> links) { - if(links == null) - return Lists.newArrayList(); - List<String> result = Lists.newLinkedList(); - for(Object obj : links) { - if(obj instanceof String) { - result.add((String) obj); - } else { - LOGGER.warn("Links is not instance of String : {}", obj.getClass().getName()); - } - } - return result; - } - - public Actor buildActor(Datasift event, Twitter twitter) { - DatasiftTwitterUser user = twitter.getUser(); - Actor actor = super.buildActor(event.getInteraction()); - if(user == null) { - user = twitter.getRetweet().getUser(); - } - - actor.setDisplayName(user.getName()); - actor.setId(formatId(Optional.fromNullable( - user.getIdStr()) - .or(Optional.of(user.getId().toString())) - .orNull())); - actor.setSummary(user.getDescription()); - try { - actor.setPublished(user.getCreatedAt()); - } catch (Exception e) { - LOGGER.warn("Exception trying to parse date : {}", e); - } - - if(user.getUrl() != null) { - actor.setUrl(user.getUrl()); - } - - Map<String, Object> extensions = new HashMap<String,Object>(); - extensions.put("location", user.getLocation()); - extensions.put("posts", user.getStatusesCount()); - extensions.put("followers", user.getFollowersCount()); - extensions.put("screenName", user.getScreenName()); - if(user.getAdditionalProperties() != null) { - extensions.put("favorites", user.getFavouritesCount()); - } - - Image profileImage = new Image(); - String profileUrl = null; - Author author = event.getInteraction().getAuthor(); - if( author != null ) - profileUrl = author.getAvatar(); - if(profileUrl == null && user.getProfileImageUrlHttps() != null) { - Object url = user.getProfileImageUrlHttps(); - if(url instanceof String) - profileUrl = (String) url; - } - if(profileUrl == null) { - profileUrl = user.getProfileImageUrl(); - } - profileImage.setUrl(profileUrl); - actor.setImage(profileImage); - - actor.setAdditionalProperty("extensions", extensions); - return actor; - } - - public void addLocationExtension(Activity activity, Twitter twitter) { - Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity); - Map<String, Object> location = Maps.newHashMap(); - double[] coordiantes = new double[] { twitter.getGeo().getLongitude(), twitter.getGeo().getLatitude() }; - Map<String, Object> coords = Maps.newHashMap(); - coords.put("coordinates", coordiantes); - coords.put("type", "geo_point"); - location.put("coordinates", coords); - extensions.put("location", location); - } - - public void addTwitterExtensions(Activity activity, Twitter twitter, Interaction interaction) { - Retweet retweet = twitter.getRetweet(); - Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity); - List<String> hashTags = Lists.newLinkedList(); - List<Object> hts = Lists.newLinkedList(); - if(twitter.getHashtags() != null) { - hts = twitter.getHashtags(); - } else if (retweet != null) { - hts = retweet.getHashtags(); - } - if(hts != null) { - for(Object ht : twitter.getHashtags()) { - if(ht instanceof String) { - hashTags.add((String) ht); - } else { - LOGGER.warn("Hashtag was not instance of String : {}", ht.getClass().getName()); - } - } - } - extensions.put("hashtags", hashTags); - - - if(retweet != null) { - Map<String, Object> rebroadcasts = Maps.newHashMap(); - rebroadcasts.put("perspectival", true); - rebroadcasts.put("count", retweet.getCount()); - extensions.put("rebroadcasts", rebroadcasts); - } - - if(interaction.getAdditionalProperties() != null) { - ArrayList<Map<String,Object>> userMentions = createUserMentions(interaction); - - if(userMentions.size() > 0) - extensions.put("user_mentions", userMentions); - } - - extensions.put("keywords", interaction.getContent()); - } - - /** - * Returns an ArrayList of all UserMentions in this interaction - * Note: The ID list and the handle lists do not necessarily correspond 1:1 for this provider - * If those lists are the same size, then they will be merged into individual UserMention - * objects. However, if they are not the same size, a new UserMention object will be created - * for each entry in both lists. - * - * @param interaction - * @return - */ - private ArrayList<Map<String,Object>> createUserMentions(Interaction interaction) { - ArrayList<String> mentions = (ArrayList<String>) interaction.getAdditionalProperties().get("mentions"); - ArrayList<Long> mentionIds = (ArrayList<Long>) interaction.getAdditionalProperties().get("mention_ids"); - ArrayList<Map<String,Object>> userMentions = new ArrayList<Map<String,Object>>(); - - if(mentions != null && !mentions.isEmpty()) { - for(int x = 0; x < mentions.size(); x ++) { - Map<String, Object> actor = new HashMap<String, Object>(); - actor.put("displayName", mentions.get(x)); - actor.put("handle", mentions.get(x)); - - userMentions.add(actor); - } - } - if(mentionIds != null && !mentionIds.isEmpty()) { - for(int x = 0; x < mentionIds.size(); x ++) { - Map<String, Object> actor = new HashMap<String, Object>(); - actor.put("id", "id:twitter:" + mentionIds.get(x)); - - userMentions.add(actor); - } - } - - return userMentions; - } - - public static String formatId(String... idparts) { - return Joiner.on(":").join(Lists.asList("id:twitter", idparts)); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/util/StreamsDatasiftMapper.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/util/StreamsDatasiftMapper.java b/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/util/StreamsDatasiftMapper.java deleted file mode 100644 index 93ab28b..0000000 --- a/streams-contrib/streams-provider-datasift/src/main/java/org/apache/streams/datasift/util/StreamsDatasiftMapper.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.streams.datasift.util; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import com.fasterxml.jackson.databind.module.SimpleModule; -import org.apache.streams.data.util.RFC3339Utils; -import org.apache.streams.jackson.StreamsJacksonMapper; -import org.joda.time.DateTime; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; - -import java.io.IOException; - -/** - * Created by sblackmon on 3/27/14. - * - * Depracated: Use StreamsJacksonMapper instead - */ -@Deprecated() -public class StreamsDatasiftMapper extends StreamsJacksonMapper { - - public static final String DATASIFT_FORMAT = "EEE, dd MMM yyyy HH:mm:ss Z"; - - public static final DateTimeFormatter DATASIFT_FORMATTER = DateTimeFormat.forPattern(DATASIFT_FORMAT); - - public static final Long getMillis(String dateTime) { - - // this function is for pig which doesn't handle exceptions well - try { - Long result = DATASIFT_FORMATTER.parseMillis(dateTime); - return result; - } catch( Exception e ) { - return null; - } - - } - - private static final StreamsDatasiftMapper INSTANCE = new StreamsDatasiftMapper(); - - public static StreamsDatasiftMapper getInstance(){ - return INSTANCE; - } - - public StreamsDatasiftMapper() { - super(); - registerModule(new SimpleModule() - { - { - addDeserializer(DateTime.class, new StdDeserializer<DateTime>(DateTime.class) { - @Override - public DateTime deserialize(JsonParser jpar, DeserializationContext context) throws IOException, JsonProcessingException { - DateTime result = null; - try { - result = DATASIFT_FORMATTER.parseDateTime(jpar.getValueAsString()); - } catch (Exception e) {} - if (result == null) { - try { - result = RFC3339Utils.getInstance().parseToUTC(jpar.getValueAsString()); - } catch (Exception e) {} - } - return result; - } - }); - } - }); - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/Datasift.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/Datasift.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/Datasift.json deleted file mode 100644 index 52d9912..0000000 --- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/Datasift.json +++ /dev/null @@ -1,462 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "type": "object", - "javaType": "org.apache.streams.datasift.Datasift", - "properties": { - "blog": { - "javaType": "org.apache.streams.datasift.blog.Blog", - "type": "object", - "dynamic": "true", - "properties": { - "author": { - "type": "object", - "javaType": "org.apache.streams.datasift.blog.BlogAuthor", - "dynamic": "true", - "properties": { - "link": { - "type": "string" - }, - "name": { - "type": "string" - } - } - }, - "content": { - "type": "string" - }, - "contenttype": { - "type": "string" - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "domain": { - "type": "string" - }, - "id": { - "type": "string" - }, - "link": { - "type": "string" - }, - "title": { - "type": "string" - }, - "type": { - "type": "string" - } - } - }, - "demographic": { - "dynamic": "true", - "properties": { - "gender": { - "type": "string" - } - } - }, - "facebook": { - "type": "object", - "$ref": "facebook/DatasiftFacebook.json" - }, - "instagram": { - "type": "object", - "$ref": "instagram/DatasiftInstagram.json" - }, - "interaction": { - "type": "object", - "$ref": "interaction/DatasiftInteraction.json" - }, - "klout": { - "javaType": "org.apache.streams.datasift.interaction.Klout", - "type": "object", - "dynamic": "true", - "properties": { - "score": { - "type": "integer" - } - } - }, - "language": { - "javaType": "org.apache.streams.datasift.language.Language", - "type": "object", - "dynamic": "true", - "properties": { - "confidence": { - "type": "integer" - }, - "tag": { - "type": "string" - }, - "tag_extended": { - "type": "string" - } - } - }, - "links": { - "javaType": "org.apache.streams.datasift.links.Links", - "type": "object", - "dynamic": "true", - "properties": { - "code": { - "type": "array", - "items": [ - { - "type": "integer" - } - ] - }, - "created_at": { - "type": "array", - "items": [ - { - "type": "string" - } - ] - }, - "hops": { - "type": "array", - "items": [ - { - "type": "string" - } - ] - }, - "meta": { - "dynamic": "true", - "javaType": "org.apache.streams.datasift.links.Meta", - "type": "object", - "properties": { - "charset": { - "type": "array", - "items": [ - { - "type": "string" - } - ] - }, - "content_type": { - "type": "array", - "items": [ - { - "type": "string" - } - ] - }, - "description": { - "type": "array", - "items": [ - { - "type": "string" - } - ] - }, - "keywords": { - "type": "array", - "items": [ - { - "type": "string" - } - ] - }, - "lang": { - "type": "array", - "items": [ - { - "type": "string" - } - ] - }, - "opengraph": { - "type": "array", - "items": [ - { - "javaType": "org.apache.streams.datasift.interaction.meta.OpenGraph", - "type": "object", - "dynamic": "true", - "properties": { - "description": { - "type": "string" - }, - "image": { - "type": "string" - }, - "site_name": { - "type": "string" - }, - "title": { - "type": "string" - }, - "type": { - "type": "string" - }, - "url": { - "type": "string" - } - } - } - ] - }, - "twitter": { - "type": "array", - "items": [ - { - "javaType": "org.apache.streams.datasift.interaction.meta.Twitter", - "type": "object", - "dynamic": "true", - "properties": { - "card": { - "type": "string" - }, - "creator": { - "type": "string" - }, - "description": { - "type": "string" - }, - "image": { - "type": "string" - }, - "player": { - "type": "string" - }, - "player_height": { - "type": "string" - }, - "player_width": { - "type": "string" - }, - "site": { - "type": "string" - }, - "title": { - "type": "string" - }, - "url": { - "type": "string" - } - } - } - ] - } - } - }, - "normalized_url": { - "type": "array", - "items": [ - { - "type": "string" - } - ] - }, - "retweet_count": { - "type": "array", - "items": [ - { - "type": "long" - } - ] - }, - "title": { - "type": "array", - "items": [ - { - "type": "string" - } - ] - }, - "url": { - "type": "array", - "items": [ - { - "type": "string" - } - ] - } - } - }, - "salience": { - "javaType": "org.apache.streams.datasift.salience.Salience", - "type": "object", - "dynamic": "true", - "properties": { - "content": { - "javaType": "org.apache.streams.datasift.salience.content.Content", - "type": "object", - "dynamic": "true", - "properties": { - "entities": { - "type": "array", - "items": [ - { - "javaType": "org.apache.streams.datasift.salience.content.Entities", - "type": "object", - "dynamic": "true", - "properties": { - "about": { - "type": "integer" - }, - "confident": { - "type": "integer" - }, - "evidence": { - "type": "integer" - }, - "label": { - "type": "string" - }, - "name": { - "type": "string" - }, - "sentiment": { - "type": "integer" - }, - "themes": { - "type": "string" - }, - "type": { - "type": "string" - } - } - } - ] - }, - "sentiment": { - "type": "integer" - } - } - }, - "title": { - "javaType": "org.apache.streams.datasift.salience.title.Title", - "type": "object", - "dynamic": "true", - "properties": { - "entities": { - "type": "array", - "items": [ - { - "javaType": "org.apache.streams.datasift.salience.title.Entities", - "type": "object", - "dynamic": "true", - "properties": { - "about": { - "type": "integer" - }, - "confident": { - "type": "integer" - }, - "evidence": { - "type": "integer" - }, - "label": { - "type": "string" - }, - "name": { - "type": "string" - }, - "sentiment": { - "type": "integer" - }, - "themes": { - "type": "string" - }, - "type": { - "type": "string" - } - } - } - ] - }, - "sentiment": { - "type": "integer" - } - } - } - } - }, - "trends": { - "javaType": "org.apache.streams.datasift.trends.Trends", - "type": "object", - "dynamic": "true", - "properties": { - "content": { - "type": "array", - "items": [ - { - "type": "string" - } - ] - }, - "source": { - "type": "array", - "items": [ - { - "type": "string" - } - ] - }, - "type": { - "type": "array", - "items": [ - { - "type": "string" - } - ] - } - } - }, - "twitter": { - "type": "object", - "$ref": "twitter/DatasiftTwitter.json" - }, - "youtube": { - "javaType": "org.apache.streams.datasift.youtube.YouTube", - "type": "object", - "dynamic": "true", - "properties": { - "author": { - "javaType": "org.apache.streams.datasift.youtube.Author", - "type": "object", - "dynamic": "true", - "properties": { - "link": { - "type": "string" - }, - "name": { - "type": "string" - } - } - }, - "commentslink": { - "type": "string" - }, - "content": { - "type": "string" - }, - "contenttype": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "id": { - "type": "string" - }, - "title": { - "type": "string" - }, - "type": { - "type": "string" - }, - "videolink": { - "type": "string" - } - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftConfiguration.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftConfiguration.json deleted file mode 100644 index 99f889e..0000000 --- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftConfiguration.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "type": "object", - "javaType": "org.apache.streams.datasift.DatasiftConfiguration", - "javaInterfaces": [ - "java.io.Serializable" - ], - "properties": { - "apiKey": { - "type": "string" - }, - "userName": { - "type": "string" - }, - "streamHash": { - "type": "array", - "minItems": 0, - "items": { - "type": "string" - } - }, - "streamCsdl": { - "type": "array", - "minItems": 0, - "items": { - "type": "string" - } - }, - "managedSources": { - "type": "array", - "minItems": 0, - "items": { - "javaType": "org.apache.streams.datasift.managed.StreamsManagedSource", - "type": "object", - "properties": { - "auth": { - "type": "array", - "items": { - "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceAuth", - "type": "object", - "properties": { - "identity_id": { - "type": "string" - }, - "parameters": { - "type": "object", - "properties": { - "value": { - "type": "string" - } - } - } - } - } - }, - "created_at": { - "type": "number" - }, - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parameters": { - "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceParams", - "type": "object", - "properties": { - "comments": { - "type": "boolean" - }, - "likes": { - "type": "boolean" - } - } - }, - "resources": { - "type": "array", - "items": [ - { - "javaType": "org.apache.streams.datasift.managed.StreamsManagedSourceResource", - "type": "object", - "properties": { - "parameters": { - "type": "object", - "properties": { - "distance": { - "type": "number" - }, - "exact_match": { - "type": "string" - }, - "foursq": { - "type": "string" - }, - "lat": { - "type": "number" - }, - "lng": { - "type": "number" - }, - "type": { - "type": "string", - "enum" : ["user", "tag", "area", "location", "popular"] - }, - "usermeta": { - "type": "string" - }, - "value": { - "type": "number" - } - } - }, - "resource_id": { - "type": "string" - } - } - } - ] - }, - "source_type": { - "type": "string", - "enum" : ["facebook_page", "googleplus", "instagram", "yammer"] - }, - "status": { - "type": "string" - } - } - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftPushConfiguration.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftPushConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftPushConfiguration.json deleted file mode 100644 index 4c9fe5c..0000000 --- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftPushConfiguration.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "type": "object", - "javaType" : "org.apache.streams.datasift.DatasiftPushConfiguration", - "extends": {"$ref":"DatasiftConfiguration.json"}, - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "streamHash": { - "type": "array", - "minItems": 1, - "items": { - "type": "string" - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftStreamConfiguration.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftStreamConfiguration.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftStreamConfiguration.json deleted file mode 100644 index 5c43fea..0000000 --- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftStreamConfiguration.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "type": "object", - "javaType" : "org.apache.streams.datasift.DatasiftStreamConfiguration", - "extends": {"$ref":"DatasiftConfiguration.json"}, - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "streamHash": { - "type": "array", - "minItems": 1, - "items": { - "type": "string" - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftWebhookData.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftWebhookData.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftWebhookData.json deleted file mode 100644 index 7f3bee6..0000000 --- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/DatasiftWebhookData.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "type": "object", - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "javaType": "org.apache.streams.datasift.DatasiftWebhookData", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "id": { - "type": "string" - }, - "hash": { - "type": "string" - }, - "hash_type": { - "type": "string" - }, - "count": { - "type": "long" - }, - "delivered_at": { - "type": "string", - "format": "date-time" - }, - "interactions": { - "type": "array", - "items": { - "type": "object", - "javaType": "org.apache.streams.datasift.Datasift", - "javaInterfaces": ["java.io.Serializable"] - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/facebook/DatasiftFacebook.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/facebook/DatasiftFacebook.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/facebook/DatasiftFacebook.json deleted file mode 100644 index 6085124..0000000 --- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/facebook/DatasiftFacebook.json +++ /dev/null @@ -1,125 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "type": "object", - "javaType" : "org.apache.streams.datasift.facebook.Facebook", - "javaInterfaces": ["java.io.Serializable"], - "properties" : - { - "application" : - { - "type" : "string" - }, - "author" : - { - "javaType" : "org.apache.streams.datasift.facebook.Author", - "type" : "object", - "dynamic" : "true", - "properties" : - { - "avatar" : - { - "type" : "string" - }, - "id" : - { - "type" : "string" - }, - "link" : - { - "type" : "string" - }, - "name" : - { - "type" : "string" - } - } - }, - "created_at" : - { - "type" : "string", - "format" : "date-time" - }, - "id" : - { - "type" : "string" - }, - "likes" : - { - "javaType" : "org.apache.streams.datasift.facebook.Likes", - "type" : "object", - "dynamic" : "true", - "properties" : - { - "count" : - { - "type" : "integer" - }, - "ids" : - { - "type" : "array", - "items" : - [ - { - "type" : "string" - } - ] - }, - "names" : - { - "type" : "array", - "items" : - [ - { - "type" : "string" - } - ] - } - } - }, - "message" : - { - "type" : "string" - }, - "source" : - { - "type" : "string" - }, - "to" : - { - "javaType" : "org.apache.streams.datasift.facebook.To", - "type" : "object", - "dynamic" : "true", - "properties" : - { - "ids" : - { - "type" : "array", - "items" : - [ - { - "type" : "string" - } - ] - }, - "names" : - { - "type" : "array", - "items" : - [ - { - "type" : "string" - } - ] - } - } - }, - "type" : - { - "type" : "string" - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/0b512d8b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/instagram/DatasiftInstagram.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/instagram/DatasiftInstagram.json b/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/instagram/DatasiftInstagram.json deleted file mode 100644 index 8c209b1..0000000 --- a/streams-contrib/streams-provider-datasift/src/main/jsonschema/org/apache/streams/datasift/instagram/DatasiftInstagram.json +++ /dev/null @@ -1,183 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "type": "object", - "javaType" : "org.apache.streams.datasift.instagram.Instagram", - "javaInterfaces": ["java.io.Serializable"], - "properties" : - { - "caption" : - { - "type" : "object", - "properties" : - { - "created_time" : - { - "type" : "number" - }, - "id" : - { - "type" : "string" - }, - "text" : - { - "type" : "string" - }, - "from" : - { - "type" : "object", - "properties" : - { - "full_name" : - { - "type" : "array", - "items" : - [ - { - "type" : "string" - } - ] - }, - "id" : - { - "type" : "string" - }, - "profile_picture" : - { - "type" : "string" - }, - "username" : - { - "type" : "string" - } - } - } - } - }, - "created_at" : - { - "type" : "string", - "format" : "date-time" - }, - "from" : - { - "javaType" : "org.apache.streams.datasift.instagram.From", - "type" : "object", - "properties" : - { - "full_name" : - { - "type" : "array", - "items" : - [ - { - "type" : "string" - } - ] - }, - "id" : - { - "type" : "string" - }, - "profile_picture" : - { - "type" : "string" - }, - "username" : - { - "type" : "string" - } - } - }, - "id" : - { - "type" : "string" - }, - "images" : - { - "type" : "object", - "properties" : - { - "low_resolution" : - { - "type" : "object" - }, - "standard_resolution" : - { - "type" : "object" - }, - "thumbnail" : - { - "type" : "object" - } - } - }, - "media" : - { - "javaType" : "org.apache.streams.datasift.instagram.Media", - "type" : "object", - "properties" : - { - "caption" : - { - "type" : "string" - }, - "created_time" : - { - "type" : "number" - }, - "filter" : - { - "type" : "string" - }, - "id" : - { - "type" : "string" - }, - "image" : - { - "type" : "string" - }, - "link" : - { - "type" : "string" - }, - "tags" : - { - "type" : "array", - "items" : - { - "type" : "string" - } - }, - "type" : - { - "type" : "string" - }, - "username" : - { - "type" : "string" - }, - "video" : - { - "type" : "string" - } - } - }, - "tags" : - { - "type" : "array", - "items" : - { - "type" : "string" - } - }, - "type" : - { - "type" : "string" - } - } -} \ No newline at end of file
