Repository: incubator-metron
Updated Branches:
  refs/heads/master 3a2ecc404 -> 32d5ff64a


METRON-413 Allow Start/End Time Range Search in Profiler Client API 
(nickwallen) closes apache/incubator-metron#249


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/32d5ff64
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/32d5ff64
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/32d5ff64

Branch: refs/heads/master
Commit: 32d5ff64af0405ac99a718896ac15511046c2575
Parents: 3a2ecc4
Author: nickwallen <n...@nickallen.org>
Authored: Fri Sep 16 12:42:59 2016 -0400
Committer: Nick Allen <n...@nickallen.org>
Committed: Fri Sep 16 12:42:59 2016 -0400

----------------------------------------------------------------------
 .../profiler/client/HBaseProfilerClient.java    |  27 ++++-
 .../metron/profiler/client/ProfilerClient.java  |  27 +++--
 .../profiler/client/stellar/GetProfile.java     |   2 +-
 .../client/HBaseProfilerClientTest.java         | 105 +++++++++++++++----
 .../metron/profiler/hbase/RowKeyBuilder.java    |  16 +--
 .../profiler/hbase/SaltyRowKeyBuilder.java      |  16 +--
 .../profiler/hbase/SaltyRowKeyBuilderTest.java  |   5 +-
 7 files changed, 149 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/32d5ff64/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/HBaseProfilerClient.java
----------------------------------------------------------------------
diff --git 
a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/HBaseProfilerClient.java
 
b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/HBaseProfilerClient.java
index cef2ea4..1d11f42 100644
--- 
a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/HBaseProfilerClient.java
+++ 
b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/HBaseProfilerClient.java
@@ -64,21 +64,40 @@ public class HBaseProfilerClient implements ProfilerClient {
   /**
    * Fetches all of the data values associated with a Profile.
    *
+   * @param clazz       The type of values stored by the profile.
    * @param profile     The name of the profile.
    * @param entity      The name of the entity.
+   * @param groups      The groups used to sort the profile data.
    * @param durationAgo How far in the past to fetch values from.
    * @param unit        The time unit of 'durationAgo'.
-   * @param groups      The groups
    * @param <T>         The type of values stored by the Profile.
-   * @return A list of profile values.
+   * @return A list of values.
    */
   @Override
-  public <T> List<T> fetch(String profile, String entity, long durationAgo, 
TimeUnit unit, Class<T> clazz, List<Object> groups) {
+  public <T> List<T> fetch(Class<T> clazz, String profile, String entity, 
List<Object> groups, long durationAgo, TimeUnit unit) {
+    long end = System.currentTimeMillis();
+    long start = end - unit.toMillis(durationAgo);
+    return fetch(clazz, profile, entity, groups, start, end);
+  }
+
+  /**
+   * Fetch the values stored in a profile based on a start and end timestamp.
+   *
+   * @param clazz   The type of values stored by the profile.
+   * @param profile The name of the profile.
+   * @param entity  The name of the entity.
+   * @param groups  The groups used to sort the profile data.
+   * @param start   The start time in epoch milliseconds.
+   * @param end     The end time in epoch milliseconds.
+   * @param <T>     The type of values stored by the profile.
+   * @return A list of values.
+   */
+  public <T> List<T> fetch(Class<T> clazz, String profile, String entity, 
List<Object> groups, long start, long end) {
     byte[] columnFamily = Bytes.toBytes(columnBuilder.getColumnFamily());
     byte[] columnQualifier = columnBuilder.getColumnQualifier("value");
 
     // find all the row keys that satisfy this fetch
-    List<byte[]> keysToFetch = rowKeyBuilder.rowKeys(profile, entity, groups, 
durationAgo, unit);
+    List<byte[]> keysToFetch = rowKeyBuilder.rowKeys(profile, entity, groups, 
start, end);
 
     // create a Get for each of the row keys
     List<Get> gets = keysToFetch

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/32d5ff64/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/ProfilerClient.java
----------------------------------------------------------------------
diff --git 
a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/ProfilerClient.java
 
b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/ProfilerClient.java
index 9cae0e9..c6a5379 100644
--- 
a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/ProfilerClient.java
+++ 
b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/ProfilerClient.java
@@ -31,15 +31,28 @@ public interface ProfilerClient {
   /**
    * Fetch the measurement values associated with a profile.
    *
-   * @param profile The name of the profile.
-   * @param entity The name of the entity.
+   * @param clazz       The type of values stored by the profile.
+   * @param profile     The name of the profile.
+   * @param entity      The name of the entity.
+   * @param groups      The groups used to sort the profile data.
    * @param durationAgo How far in the past to fetch values from.
-   * @param unit The time unit of 'durationAgo'.
-   * @param clazz The type of values stored by the profile.
-   * @param groups The groups used to sort the profile data.
-   * @param <T> The type of values stored by the Profile.
+   * @param unit        The time unit of 'durationAgo'.
+   * @param <T>         The type of values stored by the Profile.
    * @return A list of values.
    */
-  <T> List<T> fetch(String profile, String entity, long durationAgo, TimeUnit 
unit, Class<T> clazz, List<Object> groups);
+  <T> List<T> fetch(Class<T> clazz, String profile, String entity, 
List<Object> groups, long durationAgo, TimeUnit unit);
 
+  /**
+   * Fetch the values stored in a profile based on a start and end timestamp.
+   *
+   * @param clazz   The type of values stored by the profile.
+   * @param profile The name of the profile.
+   * @param entity  The name of the entity.
+   * @param groups  The groups used to sort the profile data.
+   * @param start   The start time in epoch milliseconds.
+   * @param end     The end time in epoch milliseconds.
+   * @param <T>     The type of values stored by the profile.
+   * @return A list of values.
+   */
+  <T> List<T> fetch(Class<T> clazz, String profile, String entity, 
List<Object> groups, long start, long end);
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/32d5ff64/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
----------------------------------------------------------------------
diff --git 
a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
 
b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
index d96419f..20546a2 100644
--- 
a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
+++ 
b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/stellar/GetProfile.java
@@ -142,7 +142,7 @@ public class GetProfile implements StellarFunction {
     TimeUnit units = TimeUnit.valueOf(unitsName);
     List<Object> groups = getGroupsArg(4, args);
 
-    return client.fetch(profile, entity, durationAgo, units, Object.class, 
groups);
+    return client.fetch(Object.class, profile, entity, groups, durationAgo, 
units);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/32d5ff64/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
----------------------------------------------------------------------
diff --git 
a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
 
b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
index 0da3f31..0076396 100644
--- 
a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
+++ 
b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/HBaseProfilerClientTest.java
@@ -57,6 +57,9 @@ public class HBaseProfilerClientTest {
 
   private static final String tableName = "profiler";
   private static final String columnFamily = "P";
+  private static final long periodDuration = 15;
+  private static final TimeUnit periodUnits = TimeUnit.MINUTES;
+  private static final int periodsPerHour = 4;
 
   private HBaseProfilerClient client;
   private HTableInterface table;
@@ -103,24 +106,89 @@ public class HBaseProfilerClientTest {
    * The client should be able to distinguish between groups and only fetch 
those in the correct group.
    */
   @Test
-  public void testFetchOneGroup() throws Exception {
+  public void testFetchWithDurationAgoAndOneGroup() throws Exception {
+    final int expectedValue = 2302;
+    final int hours = 2;
+    final int count = hours * periodsPerHour;
+    final long startTime = System.currentTimeMillis() - 
TimeUnit.HOURS.toMillis(hours);
+
+    // setup - write two groups of measurements - 'weekends' and 'weekdays'
+    ProfileMeasurement m = new ProfileMeasurement("profile1", "entity1", 
startTime, periodDuration, periodUnits);
+    profileWriter.write(m, count, Arrays.asList("weekdays"), val -> 
expectedValue);
+    profileWriter.write(m, count, Arrays.asList("weekends"), val -> 0);
+
+    // execute
+    List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", 
Arrays.asList("weekdays"), hours, TimeUnit.HOURS);
+
+    // validate
+    assertEquals(count, results.size());
+    results.forEach(actual -> assertEquals(expectedValue, (int) actual));
+  }
+
+  /**
+   * Attempt to fetch a group that does not exist.
+   */
+  @Test
+  public void testFetchWithDurationAgoAndNoGroup() {
 
     // setup
-    final long periodDuration = 15;
-    final TimeUnit periodUnits = TimeUnit.MINUTES;
-    final int periodsPerHour = 4;
     final int expectedValue = 2302;
     final int hours = 2;
-    final int count = hours * periodsPerHour;
     final long startTime = System.currentTimeMillis() - 
TimeUnit.HOURS.toMillis(hours);
 
     // create two groups of measurements - one on weekdays and one on weekends
     ProfileMeasurement m = new ProfileMeasurement("profile1", "entity1", 
startTime, periodDuration, periodUnits);
+    profileWriter.write(m, hours * periodsPerHour, Arrays.asList("weekdays"), 
val -> expectedValue);
+    profileWriter.write(m, hours * periodsPerHour, Arrays.asList("weekends"), 
val -> 0);
+
+    // execute
+    List<Object> doesNotExist = Arrays.asList("does-not-exist");
+    List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", 
doesNotExist, hours, TimeUnit.HOURS);
+
+    // validate
+    assertEquals(0, results.size());
+  }
+
+  /**
+   * Profile data only within 'milliseconds ago' should be fetched.  Data 
outside of that time horizon should
+   * not be fetched.
+   */
+  @Test
+  public void testFetchWithDurationAgoAndOutsideTimeWindow() throws Exception {
+    final int hours = 2;
+    final List<Object> group = Arrays.asList("weekends");
+    final long startTime = System.currentTimeMillis() - 
TimeUnit.DAYS.toMillis(1);
+
+    // setup - write some values to read later
+    ProfileMeasurement m = new ProfileMeasurement("profile1", "entity1", 
startTime, periodDuration, periodUnits);
+    profileWriter.write(m, hours * periodsPerHour, group, val -> 1000);
+
+    // execute
+    List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", 
group, 2, TimeUnit.MILLISECONDS);
+
+    // validate - there should NOT be any results from just 2 milliseconds ago
+    assertEquals(0, results.size());
+  }
+
+  /**
+   * The client should be able to distinguish between groups and only fetch 
those in the correct group.
+   */
+  @Test
+  public void testFetchWithStartEndAndOneGroup() throws Exception {
+    final int periodsPerHour = 4;
+    final int expectedValue = 2302;
+    final int hours = 2;
+    final int count = hours * periodsPerHour;
+    final long endTime = System.currentTimeMillis();
+    final long startTime = endTime - TimeUnit.HOURS.toMillis(hours);
+
+    // setup - write two groups of measurements - 'weekends' and 'weekdays'
+    ProfileMeasurement m = new ProfileMeasurement("profile1", "entity1", 
startTime, periodDuration, periodUnits);
     profileWriter.write(m, count, Arrays.asList("weekdays"), val -> 
expectedValue);
     profileWriter.write(m, count, Arrays.asList("weekends"), val -> 0);
 
     // execute
-    List<Integer> results = client.fetch("profile1", "entity1", hours, 
TimeUnit.HOURS, Integer.class, Arrays.asList("weekdays"));
+    List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", 
Arrays.asList("weekdays"), startTime, endTime);
 
     // validate
     assertEquals(count, results.size());
@@ -131,16 +199,15 @@ public class HBaseProfilerClientTest {
    * Attempt to fetch a group that does not exist.
    */
   @Test
-  public void testFetchNoGroup() {
+  public void testFetchWithStartEndAndNoGroup() {
 
     // setup
-    final long periodDuration = 15;
-    final TimeUnit periodUnits = TimeUnit.MINUTES;
     final int periodsPerHour = 4;
     final int expectedValue = 2302;
     final int hours = 2;
     final int count = hours * periodsPerHour;
-    final long startTime = System.currentTimeMillis() - 
TimeUnit.HOURS.toMillis(hours);
+    final long endTime = System.currentTimeMillis();
+    final long startTime = endTime - TimeUnit.HOURS.toMillis(hours);
 
     // create two groups of measurements - one on weekdays and one on weekends
     ProfileMeasurement m = new ProfileMeasurement("profile1", "entity1", 
startTime, periodDuration, periodUnits);
@@ -149,7 +216,7 @@ public class HBaseProfilerClientTest {
 
     // execute
     List<Object> doesNotExist = Arrays.asList("does-not-exist");
-    List<Integer> results = client.fetch("profile1", "entity1", hours, 
TimeUnit.HOURS, Integer.class, doesNotExist);
+    List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", 
doesNotExist, startTime, endTime);
 
     // validate
     assertEquals(0, results.size());
@@ -160,23 +227,23 @@ public class HBaseProfilerClientTest {
    * not be fetched.
    */
   @Test
-  public void testFetchOutsideTimeWindow() throws Exception {
-    final long periodDuration = 15;
-    final TimeUnit periodUnits = TimeUnit.MINUTES;
-    final int periodsPerHour = 4;
+  public void testFetchWithStartEndAndOutsideTimeWindow() throws Exception {
+
     final int hours = 2;
     int numberToWrite = hours * periodsPerHour;
     final List<Object> group = Arrays.asList("weekends");
-    final long startTime = System.currentTimeMillis() - 
TimeUnit.DAYS.toMillis(1);
+    final long measurementTime = System.currentTimeMillis() - 
TimeUnit.DAYS.toMillis(1);
 
     // setup - write some values to read later
-    ProfileMeasurement m = new ProfileMeasurement("profile1", "entity1", 
startTime, periodDuration, periodUnits);
+    ProfileMeasurement m = new ProfileMeasurement("profile1", "entity1", 
measurementTime, periodDuration, periodUnits);
     profileWriter.write(m, numberToWrite, group, val -> 1000);
 
     // execute
-    List<Integer> results = client.fetch("profile1", "entity1", 2, 
TimeUnit.MILLISECONDS, Integer.class, group);
+    final long endFetchAt = System.currentTimeMillis();
+    final long startFetchAt = endFetchAt - TimeUnit.MILLISECONDS.toMillis(30);
+    List<Integer> results = client.fetch(Integer.class, "profile1", "entity1", 
group, startFetchAt, endFetchAt);
 
     // validate - there should NOT be any results from just 2 milliseconds ago
     assertEquals(0, results.size());
   }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/32d5ff64/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/RowKeyBuilder.java
----------------------------------------------------------------------
diff --git 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/RowKeyBuilder.java
 
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/RowKeyBuilder.java
index 5223ee6..1ce4906 100644
--- 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/RowKeyBuilder.java
+++ 
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/RowKeyBuilder.java
@@ -34,7 +34,7 @@ public interface RowKeyBuilder extends Serializable {
 
   /**
    * Build a row key for a given ProfileMeasurement.
-   * <p>
+   * 
    * This method is useful when writing ProfileMeasurements to HBase.
    *
    * @param measurement The profile measurement.
@@ -46,15 +46,15 @@ public interface RowKeyBuilder extends Serializable {
   /**
    * Builds a list of row keys necessary to retrieve a profile's measurements 
over
    * a time horizon.
-   * <p>
+   *
    * This method is useful when attempting to read ProfileMeasurements stored 
in HBase.
    *
-   * @param profile     The name of the profile.
-   * @param entity      The name of the entity.
-   * @param groups      The group(s) used to sort the profile data.
-   * @param durationAgo How long ago?
-   * @param unit        The time units of how long ago.
+   * @param profile The name of the profile.
+   * @param entity The name of the entity.
+   * @param groups The group(s) used to sort the profile data.
+   * @param start When the time horizon starts in epoch milliseconds.
+   * @param end When the time horizon ends in epoch milliseconds.
    * @return All of the row keys necessary to retrieve the profile 
measurements.
    */
-  List<byte[]> rowKeys(String profile, String entity, List<Object> groups, 
long durationAgo, TimeUnit unit);
+  List<byte[]> rowKeys(String profile, String entity, List<Object> groups, 
long start, long end);
 }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/32d5ff64/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilder.java
----------------------------------------------------------------------
diff --git 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilder.java
 
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilder.java
index d1c48f8..3b12472 100644
--- 
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilder.java
+++ 
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilder.java
@@ -77,21 +77,21 @@ public class SaltyRowKeyBuilder implements RowKeyBuilder {
    * @param profile The name of the profile.
    * @param entity The name of the entity.
    * @param groups The group(s) used to sort the profile data.
-   * @param durationAgo How long ago?
-   * @param unit The time units of how long ago.
+   * @param start When the time horizon starts in epoch milliseconds.
+   * @param end When the time horizon ends in epoch milliseconds.
    * @return All of the row keys necessary to retrieve the profile 
measurements.
    */
   @Override
-  public List<byte[]> rowKeys(String profile, String entity, List<Object> 
groups, long durationAgo, TimeUnit unit) {
+  public List<byte[]> rowKeys(String profile, String entity, List<Object> 
groups, long start, long end) {
     List<byte[]> rowKeys = new ArrayList<>();
 
-    // find the time horizon
-    long endTime = System.currentTimeMillis();
-    long startTime = endTime - unit.toMillis(durationAgo);
+    // be forgiving of out-of-order start and end times; order is critical to 
this algorithm
+    end = Math.max(start, end);
+    start = Math.min(start, end);
 
     // find the starting period and advance until the end time is reached
-    ProfilePeriod period = new ProfilePeriod(startTime, periodDurationMillis, 
TimeUnit.MILLISECONDS);
-    while(period.getStartTimeMillis() <= endTime) {
+    ProfilePeriod period = new ProfilePeriod(start, periodDurationMillis, 
TimeUnit.MILLISECONDS);
+    while(period.getStartTimeMillis() <= end) {
 
       byte[] k = rowKey(profile, entity, period, groups);
       rowKeys.add(k);

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/32d5ff64/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java
----------------------------------------------------------------------
diff --git 
a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java
 
b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java
index b4ebcf2..56f1d51 100644
--- 
a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java
+++ 
b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilderTest.java
@@ -219,7 +219,8 @@ public class SaltyRowKeyBuilderTest {
     rowKeyBuilder = new SaltyRowKeyBuilder(saltDivisor, periodDuration, 
periodUnits);
 
     // a dummy profile measurement
-    long oldest = System.currentTimeMillis() - 
TimeUnit.HOURS.toMillis(hoursAgo);
+    long now = System.currentTimeMillis();
+    long oldest = now - TimeUnit.HOURS.toMillis(hoursAgo);
     ProfileMeasurement m = new ProfileMeasurement("profile", "entity", oldest, 
periodDuration, periodUnits);
     m.setValue(22);
 
@@ -237,7 +238,7 @@ public class SaltyRowKeyBuilderTest {
     }
 
     // execute
-    List<byte[]> actualKeys = 
rowKeyBuilder.rowKeys(measurement.getProfileName(), measurement.getEntity(), 
groups, hoursAgo, TimeUnit.HOURS);
+    List<byte[]> actualKeys = 
rowKeyBuilder.rowKeys(measurement.getProfileName(), measurement.getEntity(), 
groups, oldest, now);
 
     // validate - expectedKeys == actualKeys
     for(int i=0; i<actualKeys.size(); i++) {

Reply via email to