[54/55] [abbrv] incubator-geode git commit: GEODE-1985: Check for index expression reevalaution using a time window

2016-10-24 Thread upthewaterspout 
GEODE-1985: Check for index expression reevalaution using a time window

Changing the logic for how to we check to see if an entry may have been
concurrently modified while an indexed query is in progress.

The new logic just has a time window, defaulting to 10 minutes. If the
entry was changed less than 10 minutes for the query started, we will
reevaluate the index expression to make sure the entry is still valid.


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/d6afb70d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/d6afb70d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/d6afb70d

Branch: refs/heads/feature/GEODE-1985
Commit: d6afb70d474e54a63523b3d3245a67e18f547c15
Parents: 24a7204
Author: Dan Smith 
Authored: Mon Oct 10 17:41:07 2016 -0700
Committer: Dan Smith 
Committed: Mon Oct 24 11:11:15 2016 -0700

--
 .../query/internal/index/IndexManager.java  | 60 ++---
 .../geode/internal/cache/GemFireCacheImpl.java  |  2 +-
 .../geode/internal/cache/LocalRegion.java   |  1 -
 .../query/internal/IndexManagerJUnitTest.java   | 60 +
 ...AbstractIndexMaintenanceIntegrationTest.java | 92 +++-
 ...aintenanceNoReevaluationIntegrationTest.java | 42 +
 .../internal/index/RangeIndexAPIJUnitTest.java  |  6 +-
 .../cache/PartitionedRegionQueryDUnitTest.java  | 37 ++--
 8 files changed, 152 insertions(+), 148 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/d6afb70d/geode-core/src/main/java/org/apache/geode/cache/query/internal/index/IndexManager.java
--
diff --git 
a/geode-core/src/main/java/org/apache/geode/cache/query/internal/index/IndexManager.java
 
b/geode-core/src/main/java/org/apache/geode/cache/query/internal/index/IndexManager.java
index 27f239e..7a05e7c 100644
--- 
a/geode-core/src/main/java/org/apache/geode/cache/query/internal/index/IndexManager.java
+++ 
b/geode-core/src/main/java/org/apache/geode/cache/query/internal/index/IndexManager.java
@@ -108,12 +108,14 @@ public class IndexManager {
   public static boolean TEST_RANGEINDEX_ONLY = false;
   public static final String INDEX_ELEMARRAY_THRESHOLD_PROP = 
"index_elemarray_threshold";
   public static final String INDEX_ELEMARRAY_SIZE_PROP = 
"index_elemarray_size";
+  public static final String IN_PROGRESS_UPDATE_WINDOW_PROP = 
"index.IN_PROGRESS_UPDATE_WINDOW_MS";
   public static final int INDEX_ELEMARRAY_THRESHOLD =
   Integer.parseInt(System.getProperty(INDEX_ELEMARRAY_THRESHOLD_PROP, 
"100"));
   public static final int INDEX_ELEMARRAY_SIZE =
   Integer.parseInt(System.getProperty(INDEX_ELEMARRAY_SIZE_PROP, "5"));
-  public final static AtomicLong SAFE_QUERY_TIME = new AtomicLong(0);
-  public static boolean ENABLE_UPDATE_IN_PROGRESS_INDEX_CALCULATION = true;
+
+  public static long IN_PROGRESS_UPDATE_WINDOW =
+  Long.getLong(IN_PROGRESS_UPDATE_WINDOW_PROP, 10 * 60 * 1000);
   /** The NULL constant */
   public static final Object NULL = new NullToken();
 
@@ -143,36 +145,9 @@ public class IndexManager {
   }
 
   /**
-   * Stores the largest combination of current time + delta If there is a 
large delta/hiccup in
-   * timings, this allows us to calculate the correct results for a query but, 
reevaluate more
-   * aggressively. But the large hiccup will eventually be rolled off as time 
is always increasing
-   * This is a fix for #47475
-   * 
-   * @param operationTime the last modified time from version tag
-   * @param currentCacheTime
-   */
-  public static boolean setIndexBufferTime(long operationTime, long 
currentCacheTime) {
-long timeDifference = currentCacheTime - operationTime;
-return setNewLargestValue(SAFE_QUERY_TIME, currentCacheTime + 
timeDifference);
-  }
-
-  /**
-   * only for test purposes This should not be called from any product code. 
Calls from product code
-   * will possibly cause continous reevaluation (performance issue) OR 
incorrect query results
-   * (functional issue)
-   **/
-  public static void resetIndexBufferTime() {
-SAFE_QUERY_TIME.set(0);
-  }
-
-  /**
-   * Calculates whether we need to reevluate the key for the region entry We 
added a way to
-   * determine whether to reevaluate an entry for query execution The method 
is to keep track of the
-   * delta and current time in a single long value The value is then used by 
the query to determine
-   * if a region entry needs to be reevaluated, based on subtracting the value 
with the query
-   * execution time. This provides a delta + some false positive time (dts) If 
the dts + last
-   * modified time of the region entry is > query start time, we can assume 
that

incubator-geode git commit: GEODE-1985: Check for index expression reevalaution using a time window

2016-10-12 Thread upthewaterspout 
Repository: incubator-geode
Updated Branches:
  refs/heads/feature/GEODE-1985 [created] 7b9a3832b


GEODE-1985: Check for index expression reevalaution using a time window

Changing the logic for how to we check to see if an entry may have been
concurrently modified while an indexed query is in progress.

The new logic just has a time window, defaulting to 10 minutes. If the
entry was changed less than 10 minutes for the query started, we will
reevaluate the index expression to make sure the entry is still valid.


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/7b9a3832
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/7b9a3832
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/7b9a3832

Branch: refs/heads/feature/GEODE-1985
Commit: 7b9a3832b7e0e28c85f7157f98cf7b90085fe843
Parents: 280d2d8
Author: Dan Smith 
Authored: Mon Oct 10 17:41:07 2016 -0700
Committer: Dan Smith 
Committed: Tue Oct 11 13:21:09 2016 -0700

--
 .../query/internal/index/IndexManager.java  | 60 ++---
 .../geode/internal/cache/GemFireCacheImpl.java  |  2 +-
 .../geode/internal/cache/LocalRegion.java   |  1 -
 .../query/internal/IndexManagerJUnitTest.java   | 57 ++---
 ...AbstractIndexMaintenanceIntegrationTest.java | 88 +++-
 ...aintenanceNoReevaluationIntegrationTest.java | 43 ++
 .../internal/index/RangeIndexAPIJUnitTest.java  |  6 +-
 .../cache/PartitionedRegionQueryDUnitTest.java  | 41 ++---
 8 files changed, 150 insertions(+), 148 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/7b9a3832/geode-core/src/main/java/org/apache/geode/cache/query/internal/index/IndexManager.java
--
diff --git 
a/geode-core/src/main/java/org/apache/geode/cache/query/internal/index/IndexManager.java
 
b/geode-core/src/main/java/org/apache/geode/cache/query/internal/index/IndexManager.java
index d12cec4..547ae46 100644
--- 
a/geode-core/src/main/java/org/apache/geode/cache/query/internal/index/IndexManager.java
+++ 
b/geode-core/src/main/java/org/apache/geode/cache/query/internal/index/IndexManager.java
@@ -103,10 +103,11 @@ public class IndexManager  {
   public static boolean TEST_RANGEINDEX_ONLY = false;
   public static final String INDEX_ELEMARRAY_THRESHOLD_PROP = 
"index_elemarray_threshold";
   public static final String INDEX_ELEMARRAY_SIZE_PROP = 
"index_elemarray_size";
+  public static final String IN_PROGRESS_UPDATE_WINDOW_PROP = 
"index.IN_PROGRESS_UPDATE_WINDOW_MS";
   public static final int INDEX_ELEMARRAY_THRESHOLD = 
Integer.parseInt(System.getProperty(INDEX_ELEMARRAY_THRESHOLD_PROP,"100"));
   public static final int INDEX_ELEMARRAY_SIZE = 
Integer.parseInt(System.getProperty(INDEX_ELEMARRAY_SIZE_PROP,"5"));
-  public final static AtomicLong SAFE_QUERY_TIME = new AtomicLong(0);
-  public static boolean ENABLE_UPDATE_IN_PROGRESS_INDEX_CALCULATION = true;
+
+  public static long IN_PROGRESS_UPDATE_WINDOW = 
Long.getLong(IN_PROGRESS_UPDATE_WINDOW_PROP, 10 * 60 * 1000);
   /** The NULL constant */
   public static final Object NULL = new NullToken();
 
@@ -137,37 +138,9 @@ public class IndexManager  {
   }

   /**
-   * Stores the largest combination of current time + delta
-   * If there is a large delta/hiccup in timings, this allows us to calculate 
the 
-   * correct results for a query but, reevaluate more aggressively.
-   * But the large hiccup will eventually be rolled off as time is always 
increasing
-   * This is a fix for #47475
-   * 
-   * @param operationTime the last modified time from version tag
-   * @param currentCacheTime
-   */
-  public static boolean setIndexBufferTime(long operationTime, long 
currentCacheTime) {
-long timeDifference = currentCacheTime - operationTime;
-return setNewLargestValue(SAFE_QUERY_TIME, currentCacheTime + 
timeDifference);
-  }
-  
-  /** only for test purposes 
-   * This should not be called from any product code.  Calls from product code 
will 
-   * possibly cause continous reevaluation (performance issue) OR
-   * incorrect query results (functional issue)
-   **/
-  public static void resetIndexBufferTime() {
-SAFE_QUERY_TIME.set(0);
-  }
-  
-  /**
-   * Calculates whether we need to reevluate the key for the region entry
-   * We added a way to determine whether to reevaluate an entry for query 
execution
-   * The method is to keep track of the delta and current time in a single 
long value
-   * The value is then used by the query to determine if a region entry needs 
to be reevaluated,
-   * based on subtracting the value with the query execution time.  This 
provides a delta + some false positive time (dts)
-