Github user smashew commented on a diff in the pull request:
https://github.com/apache/incubator-streams/pull/13#discussion_r12543712
--- Diff:
streams-contrib/streams-processor-urls/src/main/java/org/apache/streams/urls/DomainSensitivity.java
---
@@ -0,0 +1,101 @@
+package org.apache.streams.urls;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ * This is a static utility helper class to obey domain sensitivity. It
cannot be
+ * instantiated and can only be referenced through the static accessor
functions
+ *
+ */
+public abstract class DomainSensitivity {
+
+ // The amount of time we want to space between domain calls
+ public static final long RECENT_DOMAINS_BACKOFF = 1000;
+ public static final long DEFAULT_STAGGER = RECENT_DOMAINS_BACKOFF / 10;
+
+ // Map to store the information of recent domains, with the last time
they were accessed.
+ private static final ConcurrentMap<String, Date> RECENT_DOMAINS = new
ConcurrentHashMap<String, Date>();
+
+ private static Timer timer;
+
+ public static void purgeAllDomainWaitTimes() {
+ RECENT_DOMAINS.clear();
+ }
+
+ public static long waitTimeForDomain(String domain) {
+ domain = domain.toLowerCase();
+
+ long toReturn = 0;
+ synchronized (DomainSensitivity.class) {
+ purgeAnyExpiredDomains();
+ // if the timer doesn't exist, then setup the timer (IE: first
time called)
+ if(timer == null)
+ setupTimer();
+ long currentTime = new Date().getTime();
+
+ if(RECENT_DOMAINS.containsKey(domain)) {
+ // find the time it wants us to wait until
+ long nextTime = RECENT_DOMAINS.get(domain).getTime();
+ long random = (long)((Math.random() *
(RECENT_DOMAINS_BACKOFF / 5))); // stagger
+
+ // back-off has been satisfied
+ if(currentTime >= nextTime)
+ RECENT_DOMAINS.put(domain, new Date(currentTime +
RECENT_DOMAINS_BACKOFF));
+ else {
+ // we are going to have to wait longer than the
back-off
+ // add the time we told them they needed to wait
+ toReturn = (nextTime - currentTime) +
RECENT_DOMAINS_BACKOFF;
+ RECENT_DOMAINS.put(domain, new Date(currentTime +
toReturn));
+ toReturn += random + 1;
+ }
+ } else {
+ // no wait
+ RECENT_DOMAINS.put(domain, new Date(currentTime +
RECENT_DOMAINS_BACKOFF));
+ }
+ } // end synchronized block
+
+ return toReturn;
+ }
+
+ /**
+ * Quick function to setup the daemon to clear domains to keep our
memory foot-print low
+ */
+ private static void setupTimer() {
+ timer = new Timer(true);
+ timer.schedule(new TimerTask() {
+ public void run() {
+ purgeAnyExpiredDomains();
+ }
+ }, RECENT_DOMAINS_BACKOFF * 2);
+ }
+
+ /**
+ * called by the timer to expire any domains
+ */
+ private static void purgeAnyExpiredDomains() {
+ // ensure this method is synchronized to get the proper information
+ synchronized (RECENT_DOMAINS) {
--- End diff --
good call. this doesn't affect execution.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
