[
https://issues.apache.org/jira/browse/STREAMS-75?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13995334#comment-13995334
]
ASF GitHub Bot commented on STREAMS-75:
---------------------------------------
Github user smashew commented on a diff in the pull request:
https://github.com/apache/incubator-streams/pull/13#discussion_r12543712
--- Diff:
streams-contrib/streams-processor-urls/src/main/java/org/apache/streams/urls/DomainSensitivity.java
---
@@ -0,0 +1,101 @@
+package org.apache.streams.urls;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ * This is a static utility helper class to obey domain sensitivity. It
cannot be
+ * instantiated and can only be referenced through the static accessor
functions
+ *
+ */
+public abstract class DomainSensitivity {
+
+ // The amount of time we want to space between domain calls
+ public static final long RECENT_DOMAINS_BACKOFF = 1000;
+ public static final long DEFAULT_STAGGER = RECENT_DOMAINS_BACKOFF / 10;
+
+ // Map to store the information of recent domains, with the last time
they were accessed.
+ private static final ConcurrentMap<String, Date> RECENT_DOMAINS = new
ConcurrentHashMap<String, Date>();
+
+ private static Timer timer;
+
+ public static void purgeAllDomainWaitTimes() {
+ RECENT_DOMAINS.clear();
+ }
+
+ public static long waitTimeForDomain(String domain) {
+ domain = domain.toLowerCase();
+
+ long toReturn = 0;
+ synchronized (DomainSensitivity.class) {
+ purgeAnyExpiredDomains();
+ // if the timer doesn't exist, then setup the timer (IE: first
time called)
+ if(timer == null)
+ setupTimer();
+ long currentTime = new Date().getTime();
+
+ if(RECENT_DOMAINS.containsKey(domain)) {
+ // find the time it wants us to wait until
+ long nextTime = RECENT_DOMAINS.get(domain).getTime();
+ long random = (long)((Math.random() *
(RECENT_DOMAINS_BACKOFF / 5))); // stagger
+
+ // back-off has been satisfied
+ if(currentTime >= nextTime)
+ RECENT_DOMAINS.put(domain, new Date(currentTime +
RECENT_DOMAINS_BACKOFF));
+ else {
+ // we are going to have to wait longer than the
back-off
+ // add the time we told them they needed to wait
+ toReturn = (nextTime - currentTime) +
RECENT_DOMAINS_BACKOFF;
+ RECENT_DOMAINS.put(domain, new Date(currentTime +
toReturn));
+ toReturn += random + 1;
+ }
+ } else {
+ // no wait
+ RECENT_DOMAINS.put(domain, new Date(currentTime +
RECENT_DOMAINS_BACKOFF));
+ }
+ } // end synchronized block
+
+ return toReturn;
+ }
+
+ /**
+ * Quick function to setup the daemon to clear domains to keep our
memory foot-print low
+ */
+ private static void setupTimer() {
+ timer = new Timer(true);
+ timer.schedule(new TimerTask() {
+ public void run() {
+ purgeAnyExpiredDomains();
+ }
+ }, RECENT_DOMAINS_BACKOFF * 2);
+ }
+
+ /**
+ * called by the timer to expire any domains
+ */
+ private static void purgeAnyExpiredDomains() {
+ // ensure this method is synchronized to get the proper information
+ synchronized (RECENT_DOMAINS) {
--- End diff --
good call. this doesn't affect execution.
> Refactor contrib/streams-processor-urls
> ---------------------------------------
>
> Key: STREAMS-75
> URL: https://issues.apache.org/jira/browse/STREAMS-75
> Project: Streams
> Issue Type: Bug
> Reporter: Matthew Hager
> Original Estimate: 4h
> Remaining Estimate: 4h
>
> refactor this class for readability and usability.
--
This message was sent by Atlassian JIRA
(v6.2#6252)
