subject:"phoenix git commit\: PHOENIX\-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread"

[37/48] phoenix git commit: PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread

2016-10-07 Thread maryannxue

PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it 
is getting used for writing by another thread


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/4c0aeb0d
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/4c0aeb0d
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/4c0aeb0d

Branch: refs/heads/calcite
Commit: 4c0aeb0d530852bc12e5fcd930e336fb19434397
Parents: 2f51568
Author: Ankit Singhal 
Authored: Mon Oct 3 18:54:51 2016 +0530
Committer: Ankit Singhal 
Committed: Mon Oct 3 18:54:51 2016 +0530

--
 .../hbase/index/table/CachingHTableFactory.java | 104 ---
 .../index/table/CoprocessorHTableFactory.java   |   6 ++
 .../hbase/index/table/HTableFactory.java|   4 +-
 .../hbase/index/write/IndexWriterUtils.java |   3 +
 .../write/ParallelWriterIndexCommitter.java |  21 ++--
 .../TrackingParallelWriterIndexCommitter.java   |  18 ++--
 .../hbase/index/write/FakeTableFactory.java |   9 +-
 .../index/write/TestCachingHTableFactory.java   |  37 ---
 .../hbase/index/write/TestIndexWriter.java  |  24 +++--
 .../index/write/TestParalleIndexWriter.java |  16 ++-
 .../write/TestParalleWriterIndexCommitter.java  |  15 ++-
 11 files changed, 197 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/4c0aeb0d/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
index 0c06e2b..d0df5b3 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
@@ -17,18 +17,30 @@
  */
 package org.apache.phoenix.hbase.index.table;
 
+import static 
org.apache.phoenix.hbase.index.write.IndexWriterUtils.HTABLE_KEEP_ALIVE_KEY;
+import static 
org.apache.phoenix.hbase.index.write.IndexWriterUtils.INDEX_WRITES_THREAD_MAX_PER_REGIONSERVER_KEY;
+
 import java.io.IOException;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.collections.map.LRUMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.util.Bytes;
-
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.phoenix.execute.DelegateHTable;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 
+import com.google.common.annotations.VisibleForTesting;;
+
 /**
  * A simple cache that just uses usual GC mechanisms to cleanup unused {@link 
HTableInterface}s.
  * When requesting an {@link HTableInterface} via {@link #getTable}, you may 
get the same table as
@@ -47,7 +59,7 @@ public class CachingHTableFactory implements HTableFactory {
   public class HTableInterfaceLRUMap extends LRUMap {
 
 public HTableInterfaceLRUMap(int cacheSize) {
-  super(cacheSize);
+  super(cacheSize, true);
 }
 
 @Override
@@ -58,12 +70,18 @@ public class CachingHTableFactory implements HTableFactory {
 + " because it was evicted from the cache.");
   }
   try {
-table.close();
+ synchronized (this) { // the whole operation of closing and checking 
the count should be atomic
+// and should not conflict with getTable()
+  if (((CachedHTableWrapper)table).getReferenceCount() <= 0) {
+table.close();
+return true;
+  }
+}
   } catch (IOException e) {
 LOG.info("Failed to correctly close HTable: " + 
Bytes.toString(table.getTableName())
 + " ignoring since being removed from queue.");
   }
-  return true;
+  return false;
 }
   }
 
@@ -73,38 +91,94 @@ public class CachingHTableFactory implements HTableFactory {
 
   private static final Log LOG = LogFactory.getLog(CachingHTableFactory.class);
   private static final String CACHE_SIZE_KEY = "index.tablefactory.cache.size";
-  private static final int DEFAULT_CACHE_SIZE = 10;
+  private static final int DEFAULT_CACHE_SIZE = 1000;
 
   private HTableFactory delegate;

phoenix git commit: PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread

2016-10-03 Thread ankit

Repository: phoenix
Updated Branches:
  refs/heads/4.8-HBase-1.2 468ca2b93 -> 8dbff1ec0


PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it 
is getting used for writing by another thread


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8dbff1ec
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8dbff1ec
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8dbff1ec

Branch: refs/heads/4.8-HBase-1.2
Commit: 8dbff1ec0725ae60636482b5086d27e3d11bd2fa
Parents: 468ca2b
Author: Ankit Singhal 
Authored: Mon Oct 3 19:15:58 2016 +0530
Committer: Ankit Singhal 
Committed: Mon Oct 3 19:15:58 2016 +0530

--
 .../hbase/index/table/CachingHTableFactory.java | 104 ---
 .../index/table/CoprocessorHTableFactory.java   |   6 ++
 .../hbase/index/table/HTableFactory.java|   4 +-
 .../hbase/index/write/IndexWriterUtils.java |   3 +
 .../write/ParallelWriterIndexCommitter.java |  21 ++--
 .../TrackingParallelWriterIndexCommitter.java   |  18 ++--
 .../hbase/index/write/FakeTableFactory.java |   9 +-
 .../index/write/TestCachingHTableFactory.java   |  37 ---
 .../hbase/index/write/TestIndexWriter.java  |  24 +++--
 .../index/write/TestParalleIndexWriter.java |  16 ++-
 .../write/TestParalleWriterIndexCommitter.java  |  15 ++-
 11 files changed, 197 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/8dbff1ec/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
index 0c06e2b..d0df5b3 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
@@ -17,18 +17,30 @@
  */
 package org.apache.phoenix.hbase.index.table;
 
+import static 
org.apache.phoenix.hbase.index.write.IndexWriterUtils.HTABLE_KEEP_ALIVE_KEY;
+import static 
org.apache.phoenix.hbase.index.write.IndexWriterUtils.INDEX_WRITES_THREAD_MAX_PER_REGIONSERVER_KEY;
+
 import java.io.IOException;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.collections.map.LRUMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.util.Bytes;
-
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.phoenix.execute.DelegateHTable;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 
+import com.google.common.annotations.VisibleForTesting;;
+
 /**
  * A simple cache that just uses usual GC mechanisms to cleanup unused {@link 
HTableInterface}s.
  * When requesting an {@link HTableInterface} via {@link #getTable}, you may 
get the same table as
@@ -47,7 +59,7 @@ public class CachingHTableFactory implements HTableFactory {
   public class HTableInterfaceLRUMap extends LRUMap {
 
 public HTableInterfaceLRUMap(int cacheSize) {
-  super(cacheSize);
+  super(cacheSize, true);
 }
 
 @Override
@@ -58,12 +70,18 @@ public class CachingHTableFactory implements HTableFactory {
 + " because it was evicted from the cache.");
   }
   try {
-table.close();
+ synchronized (this) { // the whole operation of closing and checking 
the count should be atomic
+// and should not conflict with getTable()
+  if (((CachedHTableWrapper)table).getReferenceCount() <= 0) {
+table.close();
+return true;
+  }
+}
   } catch (IOException e) {
 LOG.info("Failed to correctly close HTable: " + 
Bytes.toString(table.getTableName())
 + " ignoring since being removed from queue.");
   }
-  return true;
+  return false;
 }
   }
 
@@ -73,38 +91,94 @@ public class CachingHTableFactory implements HTableFactory {
 
   private static final Log LOG = LogFactory.getLog(CachingHTableFactory.class);
   private static final String CACHE_SIZE_KEY = "index.tablefactory.cache.size";
-  private static final int DEFAULT_CACHE_SIZE = 10;
+  private static final

phoenix git commit: PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread

2016-10-03 Thread ankit

Repository: phoenix
Updated Branches:
  refs/heads/4.8-HBase-1.1 56eb4bd28 -> be90ecf24


PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it 
is getting used for writing by another thread


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/be90ecf2
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/be90ecf2
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/be90ecf2

Branch: refs/heads/4.8-HBase-1.1
Commit: be90ecf24db1064133fbe726e0d508a629d5d74d
Parents: 56eb4bd
Author: Ankit Singhal 
Authored: Mon Oct 3 19:14:47 2016 +0530
Committer: Ankit Singhal 
Committed: Mon Oct 3 19:14:47 2016 +0530

--
 .../hbase/index/table/CachingHTableFactory.java | 104 ---
 .../index/table/CoprocessorHTableFactory.java   |   6 ++
 .../hbase/index/table/HTableFactory.java|   4 +-
 .../hbase/index/write/IndexWriterUtils.java |   3 +
 .../write/ParallelWriterIndexCommitter.java |  21 ++--
 .../TrackingParallelWriterIndexCommitter.java   |  18 ++--
 .../hbase/index/write/FakeTableFactory.java |   9 +-
 .../index/write/TestCachingHTableFactory.java   |  37 ---
 .../hbase/index/write/TestIndexWriter.java  |  24 +++--
 .../index/write/TestParalleIndexWriter.java |  16 ++-
 .../write/TestParalleWriterIndexCommitter.java  |  15 ++-
 11 files changed, 197 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/be90ecf2/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
index 0c06e2b..d0df5b3 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
@@ -17,18 +17,30 @@
  */
 package org.apache.phoenix.hbase.index.table;
 
+import static 
org.apache.phoenix.hbase.index.write.IndexWriterUtils.HTABLE_KEEP_ALIVE_KEY;
+import static 
org.apache.phoenix.hbase.index.write.IndexWriterUtils.INDEX_WRITES_THREAD_MAX_PER_REGIONSERVER_KEY;
+
 import java.io.IOException;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.collections.map.LRUMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.util.Bytes;
-
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.phoenix.execute.DelegateHTable;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 
+import com.google.common.annotations.VisibleForTesting;;
+
 /**
  * A simple cache that just uses usual GC mechanisms to cleanup unused {@link 
HTableInterface}s.
  * When requesting an {@link HTableInterface} via {@link #getTable}, you may 
get the same table as
@@ -47,7 +59,7 @@ public class CachingHTableFactory implements HTableFactory {
   public class HTableInterfaceLRUMap extends LRUMap {
 
 public HTableInterfaceLRUMap(int cacheSize) {
-  super(cacheSize);
+  super(cacheSize, true);
 }
 
 @Override
@@ -58,12 +70,18 @@ public class CachingHTableFactory implements HTableFactory {
 + " because it was evicted from the cache.");
   }
   try {
-table.close();
+ synchronized (this) { // the whole operation of closing and checking 
the count should be atomic
+// and should not conflict with getTable()
+  if (((CachedHTableWrapper)table).getReferenceCount() <= 0) {
+table.close();
+return true;
+  }
+}
   } catch (IOException e) {
 LOG.info("Failed to correctly close HTable: " + 
Bytes.toString(table.getTableName())
 + " ignoring since being removed from queue.");
   }
-  return true;
+  return false;
 }
   }
 
@@ -73,38 +91,94 @@ public class CachingHTableFactory implements HTableFactory {
 
   private static final Log LOG = LogFactory.getLog(CachingHTableFactory.class);
   private static final String CACHE_SIZE_KEY = "index.tablefactory.cache.size";
-  private static final int DEFAULT_CACHE_SIZE = 10;
+  private static final

phoenix git commit: PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread

2016-10-03 Thread ankit

Repository: phoenix
Updated Branches:
  refs/heads/4.8-HBase-1.0 afe8591cc -> 0df931ad8


PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it 
is getting used for writing by another thread


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/0df931ad
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/0df931ad
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/0df931ad

Branch: refs/heads/4.8-HBase-1.0
Commit: 0df931ad8e62a0af6694f6ba3cd62e77aeecb1b7
Parents: afe8591
Author: Ankit Singhal 
Authored: Mon Oct 3 19:11:15 2016 +0530
Committer: Ankit Singhal 
Committed: Mon Oct 3 19:11:15 2016 +0530

--
 .../hbase/index/table/CachingHTableFactory.java | 104 ---
 .../index/table/CoprocessorHTableFactory.java   |   6 ++
 .../hbase/index/table/HTableFactory.java|   4 +-
 .../hbase/index/write/IndexWriterUtils.java |   3 +
 .../write/ParallelWriterIndexCommitter.java |  17 +--
 .../TrackingParallelWriterIndexCommitter.java   |  18 ++--
 .../hbase/index/write/FakeTableFactory.java |   9 +-
 .../index/write/TestCachingHTableFactory.java   |  37 ---
 .../hbase/index/write/TestIndexWriter.java  |  24 +++--
 .../index/write/TestParalleIndexWriter.java |  16 ++-
 .../write/TestParalleWriterIndexCommitter.java  |  15 ++-
 11 files changed, 197 insertions(+), 56 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/0df931ad/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
index 0c06e2b..d0df5b3 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
@@ -17,18 +17,30 @@
  */
 package org.apache.phoenix.hbase.index.table;
 
+import static 
org.apache.phoenix.hbase.index.write.IndexWriterUtils.HTABLE_KEEP_ALIVE_KEY;
+import static 
org.apache.phoenix.hbase.index.write.IndexWriterUtils.INDEX_WRITES_THREAD_MAX_PER_REGIONSERVER_KEY;
+
 import java.io.IOException;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.collections.map.LRUMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.util.Bytes;
-
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.phoenix.execute.DelegateHTable;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 
+import com.google.common.annotations.VisibleForTesting;;
+
 /**
  * A simple cache that just uses usual GC mechanisms to cleanup unused {@link 
HTableInterface}s.
  * When requesting an {@link HTableInterface} via {@link #getTable}, you may 
get the same table as
@@ -47,7 +59,7 @@ public class CachingHTableFactory implements HTableFactory {
   public class HTableInterfaceLRUMap extends LRUMap {
 
 public HTableInterfaceLRUMap(int cacheSize) {
-  super(cacheSize);
+  super(cacheSize, true);
 }
 
 @Override
@@ -58,12 +70,18 @@ public class CachingHTableFactory implements HTableFactory {
 + " because it was evicted from the cache.");
   }
   try {
-table.close();
+ synchronized (this) { // the whole operation of closing and checking 
the count should be atomic
+// and should not conflict with getTable()
+  if (((CachedHTableWrapper)table).getReferenceCount() <= 0) {
+table.close();
+return true;
+  }
+}
   } catch (IOException e) {
 LOG.info("Failed to correctly close HTable: " + 
Bytes.toString(table.getTableName())
 + " ignoring since being removed from queue.");
   }
-  return true;
+  return false;
 }
   }
 
@@ -73,38 +91,94 @@ public class CachingHTableFactory implements HTableFactory {
 
   private static final Log LOG = LogFactory.getLog(CachingHTableFactory.class);
   private static final String CACHE_SIZE_KEY = "index.tablefactory.cache.size";
-  private static final int DEFAULT_CACHE_SIZE = 10;
+  private static final

phoenix git commit: PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread

2016-10-03 Thread ankit

Repository: phoenix
Updated Branches:
  refs/heads/4.8-HBase-0.98 54bc83839 -> 41ecf71b1


PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it 
is getting used for writing by another thread


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/41ecf71b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/41ecf71b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/41ecf71b

Branch: refs/heads/4.8-HBase-0.98
Commit: 41ecf71b1d4bb49ec41dec6609d06812dd00bb94
Parents: 54bc838
Author: Ankit Singhal 
Authored: Mon Oct 3 19:07:55 2016 +0530
Committer: Ankit Singhal 
Committed: Mon Oct 3 19:07:55 2016 +0530

--
 .../hbase/index/table/CachingHTableFactory.java | 104 ---
 .../index/table/CoprocessorHTableFactory.java   |   6 ++
 .../hbase/index/table/HTableFactory.java|   4 +-
 .../hbase/index/write/IndexWriterUtils.java |   3 +
 .../write/ParallelWriterIndexCommitter.java |  16 ++-
 .../TrackingParallelWriterIndexCommitter.java   |  18 ++--
 .../hbase/index/write/FakeTableFactory.java |   9 +-
 .../index/write/TestCachingHTableFactory.java   |  37 ---
 .../hbase/index/write/TestIndexWriter.java  |  24 +++--
 .../index/write/TestParalleIndexWriter.java |  16 ++-
 .../write/TestParalleWriterIndexCommitter.java  |  15 ++-
 11 files changed, 197 insertions(+), 55 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/41ecf71b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
index 0c06e2b..d0df5b3 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
@@ -17,18 +17,30 @@
  */
 package org.apache.phoenix.hbase.index.table;
 
+import static 
org.apache.phoenix.hbase.index.write.IndexWriterUtils.HTABLE_KEEP_ALIVE_KEY;
+import static 
org.apache.phoenix.hbase.index.write.IndexWriterUtils.INDEX_WRITES_THREAD_MAX_PER_REGIONSERVER_KEY;
+
 import java.io.IOException;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.collections.map.LRUMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.util.Bytes;
-
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.phoenix.execute.DelegateHTable;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 
+import com.google.common.annotations.VisibleForTesting;;
+
 /**
  * A simple cache that just uses usual GC mechanisms to cleanup unused {@link 
HTableInterface}s.
  * When requesting an {@link HTableInterface} via {@link #getTable}, you may 
get the same table as
@@ -47,7 +59,7 @@ public class CachingHTableFactory implements HTableFactory {
   public class HTableInterfaceLRUMap extends LRUMap {
 
 public HTableInterfaceLRUMap(int cacheSize) {
-  super(cacheSize);
+  super(cacheSize, true);
 }
 
 @Override
@@ -58,12 +70,18 @@ public class CachingHTableFactory implements HTableFactory {
 + " because it was evicted from the cache.");
   }
   try {
-table.close();
+ synchronized (this) { // the whole operation of closing and checking 
the count should be atomic
+// and should not conflict with getTable()
+  if (((CachedHTableWrapper)table).getReferenceCount() <= 0) {
+table.close();
+return true;
+  }
+}
   } catch (IOException e) {
 LOG.info("Failed to correctly close HTable: " + 
Bytes.toString(table.getTableName())
 + " ignoring since being removed from queue.");
   }
-  return true;
+  return false;
 }
   }
 
@@ -73,38 +91,94 @@ public class CachingHTableFactory implements HTableFactory {
 
   private static final Log LOG = LogFactory.getLog(CachingHTableFactory.class);
   private static final String CACHE_SIZE_KEY = "index.tablefactory.cache.size";
-  private static final int DEFAULT_CACHE_SIZE = 10;
+  private static

phoenix git commit: PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread

2016-10-03 Thread ankit

Repository: phoenix
Updated Branches:
  refs/heads/4.x-HBase-0.98 c4abc7cf2 -> fc01284b1


PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it 
is getting used for writing by another thread


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/fc01284b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/fc01284b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/fc01284b

Branch: refs/heads/4.x-HBase-0.98
Commit: fc01284b16e02b849d200d594bc42c01aa915e21
Parents: c4abc7c
Author: Ankit Singhal 
Authored: Mon Oct 3 19:03:56 2016 +0530
Committer: Ankit Singhal 
Committed: Mon Oct 3 19:03:56 2016 +0530

--
 .../hbase/index/table/CachingHTableFactory.java | 104 ---
 .../index/table/CoprocessorHTableFactory.java   |   6 ++
 .../hbase/index/table/HTableFactory.java|   4 +-
 .../hbase/index/write/IndexWriterUtils.java |   3 +
 .../write/ParallelWriterIndexCommitter.java |  16 ++-
 .../TrackingParallelWriterIndexCommitter.java   |  18 ++--
 .../hbase/index/write/FakeTableFactory.java |   9 +-
 .../index/write/TestCachingHTableFactory.java   |  37 ---
 .../hbase/index/write/TestIndexWriter.java  |  24 +++--
 .../index/write/TestParalleIndexWriter.java |  16 ++-
 .../write/TestParalleWriterIndexCommitter.java  |  15 ++-
 11 files changed, 197 insertions(+), 55 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc01284b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
index 0c06e2b..d0df5b3 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
@@ -17,18 +17,30 @@
  */
 package org.apache.phoenix.hbase.index.table;
 
+import static 
org.apache.phoenix.hbase.index.write.IndexWriterUtils.HTABLE_KEEP_ALIVE_KEY;
+import static 
org.apache.phoenix.hbase.index.write.IndexWriterUtils.INDEX_WRITES_THREAD_MAX_PER_REGIONSERVER_KEY;
+
 import java.io.IOException;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.collections.map.LRUMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.util.Bytes;
-
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.phoenix.execute.DelegateHTable;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 
+import com.google.common.annotations.VisibleForTesting;;
+
 /**
  * A simple cache that just uses usual GC mechanisms to cleanup unused {@link 
HTableInterface}s.
  * When requesting an {@link HTableInterface} via {@link #getTable}, you may 
get the same table as
@@ -47,7 +59,7 @@ public class CachingHTableFactory implements HTableFactory {
   public class HTableInterfaceLRUMap extends LRUMap {
 
 public HTableInterfaceLRUMap(int cacheSize) {
-  super(cacheSize);
+  super(cacheSize, true);
 }
 
 @Override
@@ -58,12 +70,18 @@ public class CachingHTableFactory implements HTableFactory {
 + " because it was evicted from the cache.");
   }
   try {
-table.close();
+ synchronized (this) { // the whole operation of closing and checking 
the count should be atomic
+// and should not conflict with getTable()
+  if (((CachedHTableWrapper)table).getReferenceCount() <= 0) {
+table.close();
+return true;
+  }
+}
   } catch (IOException e) {
 LOG.info("Failed to correctly close HTable: " + 
Bytes.toString(table.getTableName())
 + " ignoring since being removed from queue.");
   }
-  return true;
+  return false;
 }
   }
 
@@ -73,38 +91,94 @@ public class CachingHTableFactory implements HTableFactory {
 
   private static final Log LOG = LogFactory.getLog(CachingHTableFactory.class);
   private static final String CACHE_SIZE_KEY = "index.tablefactory.cache.size";
-  private static final int DEFAULT_CACHE_SIZE = 10;
+  private static

phoenix git commit: PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread

2016-10-03 Thread ankit

Repository: phoenix
Updated Branches:
  refs/heads/4.x-HBase-1.1 2c83fae49 -> 5db33c511


PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it 
is getting used for writing by another thread


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/5db33c51
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/5db33c51
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/5db33c51

Branch: refs/heads/4.x-HBase-1.1
Commit: 5db33c5119f49ed01bc6ea3e97124e64cbd2d022
Parents: 2c83fae
Author: Ankit Singhal 
Authored: Mon Oct 3 18:59:08 2016 +0530
Committer: Ankit Singhal 
Committed: Mon Oct 3 18:59:08 2016 +0530

--
 .../hbase/index/table/CachingHTableFactory.java | 104 ---
 .../index/table/CoprocessorHTableFactory.java   |   6 ++
 .../hbase/index/table/HTableFactory.java|   4 +-
 .../hbase/index/write/IndexWriterUtils.java |   3 +
 .../write/ParallelWriterIndexCommitter.java |  21 ++--
 .../TrackingParallelWriterIndexCommitter.java   |  18 ++--
 .../hbase/index/write/FakeTableFactory.java |   9 +-
 .../index/write/TestCachingHTableFactory.java   |  37 ---
 .../hbase/index/write/TestIndexWriter.java  |  24 +++--
 .../index/write/TestParalleIndexWriter.java |  16 ++-
 .../write/TestParalleWriterIndexCommitter.java  |  15 ++-
 11 files changed, 197 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/5db33c51/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
index 0c06e2b..d0df5b3 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
@@ -17,18 +17,30 @@
  */
 package org.apache.phoenix.hbase.index.table;
 
+import static 
org.apache.phoenix.hbase.index.write.IndexWriterUtils.HTABLE_KEEP_ALIVE_KEY;
+import static 
org.apache.phoenix.hbase.index.write.IndexWriterUtils.INDEX_WRITES_THREAD_MAX_PER_REGIONSERVER_KEY;
+
 import java.io.IOException;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.collections.map.LRUMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.util.Bytes;
-
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.phoenix.execute.DelegateHTable;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 
+import com.google.common.annotations.VisibleForTesting;;
+
 /**
  * A simple cache that just uses usual GC mechanisms to cleanup unused {@link 
HTableInterface}s.
  * When requesting an {@link HTableInterface} via {@link #getTable}, you may 
get the same table as
@@ -47,7 +59,7 @@ public class CachingHTableFactory implements HTableFactory {
   public class HTableInterfaceLRUMap extends LRUMap {
 
 public HTableInterfaceLRUMap(int cacheSize) {
-  super(cacheSize);
+  super(cacheSize, true);
 }
 
 @Override
@@ -58,12 +70,18 @@ public class CachingHTableFactory implements HTableFactory {
 + " because it was evicted from the cache.");
   }
   try {
-table.close();
+ synchronized (this) { // the whole operation of closing and checking 
the count should be atomic
+// and should not conflict with getTable()
+  if (((CachedHTableWrapper)table).getReferenceCount() <= 0) {
+table.close();
+return true;
+  }
+}
   } catch (IOException e) {
 LOG.info("Failed to correctly close HTable: " + 
Bytes.toString(table.getTableName())
 + " ignoring since being removed from queue.");
   }
-  return true;
+  return false;
 }
   }
 
@@ -73,38 +91,94 @@ public class CachingHTableFactory implements HTableFactory {
 
   private static final Log LOG = LogFactory.getLog(CachingHTableFactory.class);
   private static final String CACHE_SIZE_KEY = "index.tablefactory.cache.size";
-  private static final int DEFAULT_CACHE_SIZE = 10;
+  private static final

phoenix git commit: PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread

2016-10-03 Thread ankit

Repository: phoenix
Updated Branches:
  refs/heads/master 2f51568a7 -> 4c0aeb0d5


PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it 
is getting used for writing by another thread


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/4c0aeb0d
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/4c0aeb0d
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/4c0aeb0d

Branch: refs/heads/master
Commit: 4c0aeb0d530852bc12e5fcd930e336fb19434397
Parents: 2f51568
Author: Ankit Singhal 
Authored: Mon Oct 3 18:54:51 2016 +0530
Committer: Ankit Singhal 
Committed: Mon Oct 3 18:54:51 2016 +0530

--
 .../hbase/index/table/CachingHTableFactory.java | 104 ---
 .../index/table/CoprocessorHTableFactory.java   |   6 ++
 .../hbase/index/table/HTableFactory.java|   4 +-
 .../hbase/index/write/IndexWriterUtils.java |   3 +
 .../write/ParallelWriterIndexCommitter.java |  21 ++--
 .../TrackingParallelWriterIndexCommitter.java   |  18 ++--
 .../hbase/index/write/FakeTableFactory.java |   9 +-
 .../index/write/TestCachingHTableFactory.java   |  37 ---
 .../hbase/index/write/TestIndexWriter.java  |  24 +++--
 .../index/write/TestParalleIndexWriter.java |  16 ++-
 .../write/TestParalleWriterIndexCommitter.java  |  15 ++-
 11 files changed, 197 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/4c0aeb0d/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
index 0c06e2b..d0df5b3 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/table/CachingHTableFactory.java
@@ -17,18 +17,30 @@
  */
 package org.apache.phoenix.hbase.index.table;
 
+import static 
org.apache.phoenix.hbase.index.write.IndexWriterUtils.HTABLE_KEEP_ALIVE_KEY;
+import static 
org.apache.phoenix.hbase.index.write.IndexWriterUtils.INDEX_WRITES_THREAD_MAX_PER_REGIONSERVER_KEY;
+
 import java.io.IOException;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.collections.map.LRUMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.util.Bytes;
-
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.phoenix.execute.DelegateHTable;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 
+import com.google.common.annotations.VisibleForTesting;;
+
 /**
  * A simple cache that just uses usual GC mechanisms to cleanup unused {@link 
HTableInterface}s.
  * When requesting an {@link HTableInterface} via {@link #getTable}, you may 
get the same table as
@@ -47,7 +59,7 @@ public class CachingHTableFactory implements HTableFactory {
   public class HTableInterfaceLRUMap extends LRUMap {
 
 public HTableInterfaceLRUMap(int cacheSize) {
-  super(cacheSize);
+  super(cacheSize, true);
 }
 
 @Override
@@ -58,12 +70,18 @@ public class CachingHTableFactory implements HTableFactory {
 + " because it was evicted from the cache.");
   }
   try {
-table.close();
+ synchronized (this) { // the whole operation of closing and checking 
the count should be atomic
+// and should not conflict with getTable()
+  if (((CachedHTableWrapper)table).getReferenceCount() <= 0) {
+table.close();
+return true;
+  }
+}
   } catch (IOException e) {
 LOG.info("Failed to correctly close HTable: " + 
Bytes.toString(table.getTableName())
 + " ignoring since being removed from queue.");
   }
-  return true;
+  return false;
 }
   }
 
@@ -73,38 +91,94 @@ public class CachingHTableFactory implements HTableFactory {
 
   private static final Log LOG = LogFactory.getLog(CachingHTableFactory.class);
   private static final String CACHE_SIZE_KEY = "index.tablefactory.cache.size";
-  private static final int DEFAULT_CACHE_SIZE = 10;
+  private static final int

[37/48] phoenix git commit: PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread

phoenix git commit: PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread

phoenix git commit: PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread

phoenix git commit: PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread

phoenix git commit: PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread

phoenix git commit: PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread

phoenix git commit: PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread

phoenix git commit: PHOENIX-3159 CachingHTableFactory may close HTable during eviction even if it is getting used for writing by another thread

8 matches

Site Navigation

Mail list logo

Footer information