[25/50] [abbrv] metron git commit: METRON-1494 Profiler Emits Messages to Kafka When Not Needed (nickwallen) closes apache/metron#967

2018-05-15 Thread otto 
METRON-1494 Profiler Emits Messages to Kafka When Not Needed (nickwallen) 
closes apache/metron#967


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/62d1a1bf
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/62d1a1bf
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/62d1a1bf

Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 62d1a1bf7e8b9b3ee2f260c358719ea5080c9045
Parents: 438893b
Author: nickwallen 
Authored: Wed Apr 11 17:57:09 2018 -0400
Committer: nickallen 
Committed: Wed Apr 11 17:57:09 2018 -0400

--
 .../metron/profiler/DefaultProfileBuilder.java  |   5 +
 .../bolt/FixedFrequencyFlushSignal.java |  13 +-
 .../metron/profiler/bolt/HBaseEmitter.java  |  12 +-
 .../metron/profiler/bolt/KafkaEmitter.java  |  78 +--
 .../profiler/bolt/ProfileSplitterBolt.java  |   5 +
 .../metron/profiler/bolt/HBaseEmitterTest.java  | 120 +++
 .../metron/profiler/bolt/KafkaEmitterTest.java  | 201 +--
 7 files changed, 358 insertions(+), 76 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/62d1a1bf/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
--
diff --git 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
index 4b564c9..66034ac 100644
--- 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
+++ 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
@@ -124,8 +124,13 @@ public class DefaultProfileBuilder implements 
ProfileBuilder, Serializable {
*/
   @Override
   public void apply(JSONObject message, long timestamp) {
+LOG.debug("Applying message to profile; profile={}, entity={}, 
timestamp={}",
+profileName, entity, timestamp);
+
 try {
   if (!isInitialized()) {
+LOG.debug("Initializing profile; profile={}, entity={}, timestamp={}",
+profileName, entity, timestamp);
 
 // execute each 'init' expression
 assign(definition.getInit(), message, "init");

http://git-wip-us.apache.org/repos/asf/metron/blob/62d1a1bf/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
--
diff --git 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
 
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
index b9f57dd..8c0a0b1 100644
--- 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
+++ 
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
@@ -94,7 +94,8 @@ public class FixedFrequencyFlushSignal implements FlushSignal 
{
 
   // set the next time to flush
   flushTime = currentTime + flushFrequency;
-  LOG.debug("Setting flush time; flushTime={}, currentTime={}, 
flushFreq={}",
+  LOG.debug("Setting flush time; '{}' ms until flush; flushTime={}, 
currentTime={}, flushFreq={}",
+  timeToNextFlush(),
   flushTime,
   currentTime,
   flushFrequency);
@@ -112,7 +113,7 @@ public class FixedFrequencyFlushSignal implements 
FlushSignal {
 boolean flush = currentTime > flushTime;
 LOG.debug("Flush={}, '{}' ms until flush; currentTime={}, flushTime={}",
 flush,
-flush ? 0 : (flushTime-currentTime),
+timeToNextFlush(),
 currentTime,
 flushTime);
 
@@ -123,4 +124,12 @@ public class FixedFrequencyFlushSignal implements 
FlushSignal {
   public long currentTimeMillis() {
 return currentTime;
   }
+
+  /**
+   * Returns the number of milliseconds to the next flush.
+   * @return The time left until the next flush.
+   */
+  private long timeToNextFlush() {
+return Math.max(0, flushTime - currentTime);
+  }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/62d1a1bf/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/HBaseEmitter.java
--
diff --git 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/HBaseEmitter.java

[39/52] [abbrv] metron git commit: METRON-1494 Profiler Emits Messages to Kafka When Not Needed (nickwallen) closes apache/metron#967

2018-04-18 Thread otto 
METRON-1494 Profiler Emits Messages to Kafka When Not Needed (nickwallen) 
closes apache/metron#967


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/62d1a1bf
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/62d1a1bf
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/62d1a1bf

Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual
Commit: 62d1a1bf7e8b9b3ee2f260c358719ea5080c9045
Parents: 438893b
Author: nickwallen 
Authored: Wed Apr 11 17:57:09 2018 -0400
Committer: nickallen 
Committed: Wed Apr 11 17:57:09 2018 -0400

--
 .../metron/profiler/DefaultProfileBuilder.java  |   5 +
 .../bolt/FixedFrequencyFlushSignal.java |  13 +-
 .../metron/profiler/bolt/HBaseEmitter.java  |  12 +-
 .../metron/profiler/bolt/KafkaEmitter.java  |  78 +--
 .../profiler/bolt/ProfileSplitterBolt.java  |   5 +
 .../metron/profiler/bolt/HBaseEmitterTest.java  | 120 +++
 .../metron/profiler/bolt/KafkaEmitterTest.java  | 201 +--
 7 files changed, 358 insertions(+), 76 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/62d1a1bf/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
--
diff --git 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
index 4b564c9..66034ac 100644
--- 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
+++ 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
@@ -124,8 +124,13 @@ public class DefaultProfileBuilder implements 
ProfileBuilder, Serializable {
*/
   @Override
   public void apply(JSONObject message, long timestamp) {
+LOG.debug("Applying message to profile; profile={}, entity={}, 
timestamp={}",
+profileName, entity, timestamp);
+
 try {
   if (!isInitialized()) {
+LOG.debug("Initializing profile; profile={}, entity={}, timestamp={}",
+profileName, entity, timestamp);
 
 // execute each 'init' expression
 assign(definition.getInit(), message, "init");

http://git-wip-us.apache.org/repos/asf/metron/blob/62d1a1bf/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
--
diff --git 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
 
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
index b9f57dd..8c0a0b1 100644
--- 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
+++ 
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
@@ -94,7 +94,8 @@ public class FixedFrequencyFlushSignal implements FlushSignal 
{
 
   // set the next time to flush
   flushTime = currentTime + flushFrequency;
-  LOG.debug("Setting flush time; flushTime={}, currentTime={}, 
flushFreq={}",
+  LOG.debug("Setting flush time; '{}' ms until flush; flushTime={}, 
currentTime={}, flushFreq={}",
+  timeToNextFlush(),
   flushTime,
   currentTime,
   flushFrequency);
@@ -112,7 +113,7 @@ public class FixedFrequencyFlushSignal implements 
FlushSignal {
 boolean flush = currentTime > flushTime;
 LOG.debug("Flush={}, '{}' ms until flush; currentTime={}, flushTime={}",
 flush,
-flush ? 0 : (flushTime-currentTime),
+timeToNextFlush(),
 currentTime,
 flushTime);
 
@@ -123,4 +124,12 @@ public class FixedFrequencyFlushSignal implements 
FlushSignal {
   public long currentTimeMillis() {
 return currentTime;
   }
+
+  /**
+   * Returns the number of milliseconds to the next flush.
+   * @return The time left until the next flush.
+   */
+  private long timeToNextFlush() {
+return Math.max(0, flushTime - currentTime);
+  }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/62d1a1bf/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/HBaseEmitter.java
--
diff --git 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/HBaseEmitter.java

metron git commit: METRON-1494 Profiler Emits Messages to Kafka When Not Needed (nickwallen) closes apache/metron#967

2018-04-11 Thread nickallen 
Repository: metron
Updated Branches:
  refs/heads/master 438893b78 -> 62d1a1bf7


METRON-1494 Profiler Emits Messages to Kafka When Not Needed (nickwallen) 
closes apache/metron#967


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/62d1a1bf
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/62d1a1bf
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/62d1a1bf

Branch: refs/heads/master
Commit: 62d1a1bf7e8b9b3ee2f260c358719ea5080c9045
Parents: 438893b
Author: nickwallen 
Authored: Wed Apr 11 17:57:09 2018 -0400
Committer: nickallen 
Committed: Wed Apr 11 17:57:09 2018 -0400

--
 .../metron/profiler/DefaultProfileBuilder.java  |   5 +
 .../bolt/FixedFrequencyFlushSignal.java |  13 +-
 .../metron/profiler/bolt/HBaseEmitter.java  |  12 +-
 .../metron/profiler/bolt/KafkaEmitter.java  |  78 +--
 .../profiler/bolt/ProfileSplitterBolt.java  |   5 +
 .../metron/profiler/bolt/HBaseEmitterTest.java  | 120 +++
 .../metron/profiler/bolt/KafkaEmitterTest.java  | 201 +--
 7 files changed, 358 insertions(+), 76 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/62d1a1bf/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
--
diff --git 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
index 4b564c9..66034ac 100644
--- 
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
+++ 
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultProfileBuilder.java
@@ -124,8 +124,13 @@ public class DefaultProfileBuilder implements 
ProfileBuilder, Serializable {
*/
   @Override
   public void apply(JSONObject message, long timestamp) {
+LOG.debug("Applying message to profile; profile={}, entity={}, 
timestamp={}",
+profileName, entity, timestamp);
+
 try {
   if (!isInitialized()) {
+LOG.debug("Initializing profile; profile={}, entity={}, timestamp={}",
+profileName, entity, timestamp);
 
 // execute each 'init' expression
 assign(definition.getInit(), message, "init");

http://git-wip-us.apache.org/repos/asf/metron/blob/62d1a1bf/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
--
diff --git 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
 
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
index b9f57dd..8c0a0b1 100644
--- 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
+++ 
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/FixedFrequencyFlushSignal.java
@@ -94,7 +94,8 @@ public class FixedFrequencyFlushSignal implements FlushSignal 
{
 
   // set the next time to flush
   flushTime = currentTime + flushFrequency;
-  LOG.debug("Setting flush time; flushTime={}, currentTime={}, 
flushFreq={}",
+  LOG.debug("Setting flush time; '{}' ms until flush; flushTime={}, 
currentTime={}, flushFreq={}",
+  timeToNextFlush(),
   flushTime,
   currentTime,
   flushFrequency);
@@ -112,7 +113,7 @@ public class FixedFrequencyFlushSignal implements 
FlushSignal {
 boolean flush = currentTime > flushTime;
 LOG.debug("Flush={}, '{}' ms until flush; currentTime={}, flushTime={}",
 flush,
-flush ? 0 : (flushTime-currentTime),
+timeToNextFlush(),
 currentTime,
 flushTime);
 
@@ -123,4 +124,12 @@ public class FixedFrequencyFlushSignal implements 
FlushSignal {
   public long currentTimeMillis() {
 return currentTime;
   }
+
+  /**
+   * Returns the number of milliseconds to the next flush.
+   * @return The time left until the next flush.
+   */
+  private long timeToNextFlush() {
+return Math.max(0, flushTime - currentTime);
+  }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/62d1a1bf/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/HBaseEmitter.java
--
diff --git 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/HBaseEmitter.java