Repository: phoenix Updated Branches: refs/heads/4.x-HBase-1.0 ea58c9092 -> ea3b8eaf6
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ea3b8eaf/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropViewIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropViewIT.java index 38e0c6c..9287eef 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropViewIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropViewIT.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; +import org.apache.phoenix.end2end.BaseHBaseManagedTimeTableReuseIT; import org.apache.phoenix.end2end.Shadower; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PropertiesUtil; http://git-wip-us.apache.org/repos/asf/phoenix/blob/ea3b8eaf/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableUpsertSelectIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableUpsertSelectIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableUpsertSelectIT.java index 65eeb20..b21655a 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableUpsertSelectIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableUpsertSelectIT.java @@ -32,11 +32,12 @@ import java.sql.SQLException; import java.util.Properties; import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; +import org.apache.phoenix.end2end.BaseHBaseManagedTimeTableReuseIT; import org.apache.phoenix.util.PropertiesUtil; import org.junit.Test; -public class SaltedTableUpsertSelectIT extends BaseHBaseManagedTimeIT { +public class SaltedTableUpsertSelectIT extends BaseHBaseManagedTimeTableReuseIT { @Test public void testUpsertIntoSaltedTableFromNormalTable() throws Exception { @@ -44,26 +45,28 @@ public class SaltedTableUpsertSelectIT extends BaseHBaseManagedTimeIT { Connection conn = DriverManager.getConnection(getUrl(), props); conn.setAutoCommit(false); try { - String ddl = "CREATE TABLE IF NOT EXISTS source" + + String source = generateRandomString(); + String ddl = "CREATE TABLE IF NOT EXISTS " + source + " (pk VARCHAR NOT NULL PRIMARY KEY, col INTEGER)"; createTestTable(getUrl(), ddl); - ddl = "CREATE TABLE IF NOT EXISTS target" + + String target = generateRandomString(); + ddl = "CREATE TABLE IF NOT EXISTS " + target + " (pk VARCHAR NOT NULL PRIMARY KEY, col INTEGER) SALT_BUCKETS=4"; createTestTable(getUrl(), ddl); - String query = "UPSERT INTO source(pk, col) VALUES(?,?)"; + String query = "UPSERT INTO " + source + "(pk, col) VALUES(?,?)"; PreparedStatement stmt = conn.prepareStatement(query); stmt.setString(1, "1"); stmt.setInt(2, 1); stmt.execute(); conn.commit(); - query = "UPSERT INTO target(pk, col) SELECT pk, col from source"; + query = "UPSERT INTO " + target + "(pk, col) SELECT pk, col from " + source; stmt = conn.prepareStatement(query); stmt.execute(); conn.commit(); - query = "SELECT * FROM target"; + query = "SELECT * FROM " + target; stmt = conn.prepareStatement(query); ResultSet rs = stmt.executeQuery(); assertTrue(rs.next()); @@ -81,26 +84,28 @@ public class SaltedTableUpsertSelectIT extends BaseHBaseManagedTimeIT { Connection conn = DriverManager.getConnection(getUrl(), props); conn.setAutoCommit(false); try { - String ddl = "CREATE TABLE IF NOT EXISTS source" + + String source = generateRandomString(); + String ddl = "CREATE TABLE IF NOT EXISTS " + source + " (pk VARCHAR NOT NULL PRIMARY KEY, col INTEGER) SALT_BUCKETS=4"; createTestTable(getUrl(), ddl); - ddl = "CREATE TABLE IF NOT EXISTS target" + + String target = generateRandomString(); + ddl = "CREATE TABLE IF NOT EXISTS " + target + " (pk VARCHAR NOT NULL PRIMARY KEY, col INTEGER)"; createTestTable(getUrl(), ddl); - String query = "UPSERT INTO source(pk, col) VALUES(?,?)"; + String query = "UPSERT INTO " + source + "(pk, col) VALUES(?,?)"; PreparedStatement stmt = conn.prepareStatement(query); stmt.setString(1, "1"); stmt.setInt(2, 1); stmt.execute(); conn.commit(); - query = "UPSERT INTO target(pk, col) SELECT pk, col from source"; + query = "UPSERT INTO " + target + "(pk, col) SELECT pk, col from " + source; stmt = conn.prepareStatement(query); stmt.execute(); conn.commit(); - analyzeTable(conn, "source"); - analyzeTable(conn, "target"); - query = "SELECT * FROM target"; + analyzeTable(conn, source); + analyzeTable(conn, target); + query = "SELECT * FROM " + target; stmt = conn.prepareStatement(query); ResultSet rs = stmt.executeQuery(); assertTrue(rs.next()); @@ -123,26 +128,28 @@ public class SaltedTableUpsertSelectIT extends BaseHBaseManagedTimeIT { Connection conn = DriverManager.getConnection(getUrl(), props); conn.setAutoCommit(false); try { - String ddl = "CREATE TABLE IF NOT EXISTS source" + + String source = generateRandomString(); + String ddl = "CREATE TABLE IF NOT EXISTS " + source + " (pk VARCHAR NOT NULL PRIMARY KEY, col INTEGER) SALT_BUCKETS=4"; createTestTable(getUrl(), ddl); - ddl = "CREATE TABLE IF NOT EXISTS target" + + String target = generateRandomString(); + ddl = "CREATE TABLE IF NOT EXISTS " + target + " (pk VARCHAR NOT NULL PRIMARY KEY, col INTEGER) SALT_BUCKETS=4"; createTestTable(getUrl(), ddl); - String query = "UPSERT INTO source(pk, col) VALUES(?,?)"; + String query = "UPSERT INTO " + source + "(pk, col) VALUES(?,?)"; PreparedStatement stmt = conn.prepareStatement(query); stmt.setString(1, "1"); stmt.setInt(2, 1); stmt.execute(); conn.commit(); - query = "UPSERT INTO target(pk, col) SELECT pk, col from source"; + query = "UPSERT INTO " + target + "(pk, col) SELECT pk, col from " + source; stmt = conn.prepareStatement(query); stmt.execute(); conn.commit(); - query = "SELECT * FROM target"; + query = "SELECT * FROM " + target; stmt = conn.prepareStatement(query); ResultSet rs = stmt.executeQuery(); assertTrue(rs.next()); @@ -160,23 +167,24 @@ public class SaltedTableUpsertSelectIT extends BaseHBaseManagedTimeIT { Connection conn = DriverManager.getConnection(getUrl(), props); conn.setAutoCommit(false); try { - String ddl = "CREATE TABLE IF NOT EXISTS source" + + String source = generateRandomString(); + String ddl = "CREATE TABLE IF NOT EXISTS " + source + " (pk VARCHAR NOT NULL PRIMARY KEY, col1 INTEGER, col2 INTEGER) SALT_BUCKETS=4"; createTestTable(getUrl(), ddl); - String query = "UPSERT INTO source(pk, col1) VALUES(?,?)"; + String query = "UPSERT INTO " + source + "(pk, col1) VALUES(?,?)"; PreparedStatement stmt = conn.prepareStatement(query); stmt.setString(1, "1"); stmt.setInt(2, 1); stmt.execute(); conn.commit(); - query = "UPSERT INTO source(pk, col2) SELECT pk, col1 from source"; + query = "UPSERT INTO " + source + "(pk, col2) SELECT pk, col1 from " + source; stmt = conn.prepareStatement(query); stmt.execute(); conn.commit(); - query = "SELECT col2 FROM source"; + query = "SELECT col2 FROM " + source; stmt = conn.prepareStatement(query); ResultSet rs = stmt.executeQuery(); assertTrue(rs.next()); @@ -193,12 +201,13 @@ public class SaltedTableUpsertSelectIT extends BaseHBaseManagedTimeIT { Connection conn = DriverManager.getConnection(getUrl(), props); conn.setAutoCommit(false); try { - String ddl = "CREATE TABLE IF NOT EXISTS source1" + + String source1 = generateRandomString(); + String ddl = "CREATE TABLE IF NOT EXISTS " + source1 + " (pk1 varchar NULL, pk2 varchar NULL, pk3 integer NOT NULL, col1 INTEGER" + " CONSTRAINT pk PRIMARY KEY (pk1, pk2, pk3)) SALT_BUCKETS=4"; createTestTable(getUrl(), ddl); - String query = "UPSERT INTO source1(pk1, pk2, pk3, col1) VALUES(?,?,?,?)"; + String query = "UPSERT INTO " + source1 + "(pk1, pk2, pk3, col1) VALUES(?,?,?,?)"; PreparedStatement stmt = conn.prepareStatement(query); stmt.setString(1, "1"); stmt.setString(2, "2"); @@ -208,12 +217,13 @@ public class SaltedTableUpsertSelectIT extends BaseHBaseManagedTimeIT { conn.commit(); conn.setAutoCommit(true); - query = "UPSERT INTO source1(pk3, col1, pk1) SELECT pk3+1, col1+1, pk2 from source1"; + query = "UPSERT INTO " + source1 + + "(pk3, col1, pk1) SELECT pk3+1, col1+1, pk2 from " + source1; stmt = conn.prepareStatement(query); stmt.execute(); conn.commit(); - analyzeTable(conn, "source1"); - query = "SELECT col1 FROM source1"; + analyzeTable(conn, source1); + query = "SELECT col1 FROM " + source1; stmt = conn.prepareStatement(query); ResultSet rs = stmt.executeQuery(); assertTrue(rs.next()); @@ -232,13 +242,14 @@ public class SaltedTableUpsertSelectIT extends BaseHBaseManagedTimeIT { Connection conn = DriverManager.getConnection(getUrl(), props); conn.setAutoCommit(false); try { - String ddl = "CREATE TABLE IF NOT EXISTS source1" + + String source1 = generateRandomString(); + String ddl = "CREATE TABLE IF NOT EXISTS " + source1 + " (pk1 varchar NULL, pk2 varchar NULL, pk3 integer NOT NULL, col1 INTEGER" + " CONSTRAINT pk PRIMARY KEY (pk1, pk2, pk3)) SALT_BUCKETS=4"; createTestTable(getUrl(), ddl); for (int i = 0; i < 1000; i++) { - String upsert = "UPSERT INTO source1(pk1, pk2, pk3, col1) VALUES (?,?,?,?)"; + String upsert = "UPSERT INTO " + source1 + "(pk1, pk2, pk3, col1) VALUES (?,?,?,?)"; PreparedStatement stmt = conn.prepareStatement(upsert); stmt.setString(1, Integer.toString(i)); stmt.setString(2, Integer.toString(i)); @@ -248,13 +259,14 @@ public class SaltedTableUpsertSelectIT extends BaseHBaseManagedTimeIT { } conn.commit(); - String ddl2 = "CREATE TABLE IF NOT EXISTS source2" + + String source2 = generateRandomString(); + String ddl2 = "CREATE TABLE IF NOT EXISTS " + source2 + " (pk1 varchar NULL, pk2 varchar NULL, pk3 integer NOT NULL, col1 INTEGER" + " CONSTRAINT pk PRIMARY KEY (pk1, pk2, pk3)) SALT_BUCKETS=4"; createTestTable(getUrl(), ddl2); for (int i = 0; i < 1000; i++) { - String upsert = "UPSERT INTO source2(pk1, pk2, pk3, col1) VALUES (?,?,?,?)"; + String upsert = "UPSERT INTO " + source2 + "(pk1, pk2, pk3, col1) VALUES (?,?,?,?)"; PreparedStatement stmt = conn.prepareStatement(upsert); stmt.setString(1, Integer.toString(i)); stmt.setString(2, Integer.toString(i)); @@ -264,16 +276,22 @@ public class SaltedTableUpsertSelectIT extends BaseHBaseManagedTimeIT { } conn.commit(); - String ddl3 = "CREATE TABLE IF NOT EXISTS dest" + + String dest = generateRandomString(); + String ddl3 = "CREATE TABLE IF NOT EXISTS " + dest + " (pk1 varchar NULL, pk2 varchar NULL, pk3 integer NOT NULL, col1 INTEGER" + " CONSTRAINT pk PRIMARY KEY (pk1, pk2, pk3)) SALT_BUCKETS=4"; createTestTable(getUrl(), ddl3); - String query = "UPSERT INTO dest(pk1, pk2, pk3, col1) SELECT S1.pk1, S1.pk2, S2.pk3, S2.col1 FROM source1 AS S1 JOIN source2 AS S2 ON S1.pk1 = S2.pk1 AND S1.pk2 = S2.pk2 AND S1.pk3 = S2.pk3"; + String query = + "UPSERT INTO " + dest + + "(pk1, pk2, pk3, col1) SELECT S1.pk1, S1.pk2, S2.pk3, S2.col1 FROM " + + source1 + + " AS S1 JOIN " + source2 + + " AS S2 ON S1.pk1 = S2.pk1 AND S1.pk2 = S2.pk2 AND S1.pk3 = S2.pk3"; conn.createStatement().execute(query); conn.commit(); - query = "SELECT COUNT(*) FROM dest"; + query = "SELECT COUNT(*) FROM " + dest; PreparedStatement stmt = conn.prepareStatement(query); ResultSet rs = stmt.executeQuery(); assertTrue(rs.next()); http://git-wip-us.apache.org/repos/asf/phoenix/blob/ea3b8eaf/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableVarLengthRowKeyIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableVarLengthRowKeyIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableVarLengthRowKeyIT.java index 53b267c..e94ae3a 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableVarLengthRowKeyIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableVarLengthRowKeyIT.java @@ -30,11 +30,14 @@ import java.sql.ResultSet; import java.util.Properties; import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; +import org.apache.phoenix.end2end.BaseHBaseManagedTimeTableReuseIT; import org.apache.phoenix.util.PropertiesUtil; import org.junit.Test; -public class SaltedTableVarLengthRowKeyIT extends BaseHBaseManagedTimeIT { +public class SaltedTableVarLengthRowKeyIT extends BaseHBaseManagedTimeTableReuseIT { + + private static final String TEST_TABLE = generateRandomString(); private static void initTableValues() throws Exception { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); @@ -42,9 +45,9 @@ public class SaltedTableVarLengthRowKeyIT extends BaseHBaseManagedTimeIT { conn.setAutoCommit(false); try { - createTestTable(getUrl(), "create table testVarcharKey " + + createTestTable(getUrl(), "create table " + TEST_TABLE + " " + " (key_string varchar not null primary key, kv integer) SALT_BUCKETS=4\n"); - String query = "UPSERT INTO testVarcharKey VALUES(?,?)"; + String query = "UPSERT INTO " + TEST_TABLE + " VALUES(?,?)"; PreparedStatement stmt = conn.prepareStatement(query); stmt.setString(1, "a"); stmt.setInt(2, 1); @@ -74,7 +77,7 @@ public class SaltedTableVarLengthRowKeyIT extends BaseHBaseManagedTimeIT { PreparedStatement stmt; ResultSet rs; - query = "SELECT * FROM testVarcharKey where key_string = 'abc'"; + query = "SELECT * FROM " + TEST_TABLE + " where key_string = 'abc'"; stmt = conn.prepareStatement(query); rs = stmt.executeQuery(); assertTrue(rs.next()); http://git-wip-us.apache.org/repos/asf/phoenix/blob/ea3b8eaf/phoenix-core/src/it/java/org/apache/phoenix/iterate/PhoenixQueryTimeoutIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/iterate/PhoenixQueryTimeoutIT.java b/phoenix-core/src/it/java/org/apache/phoenix/iterate/PhoenixQueryTimeoutIT.java index 8399362..6bf3790 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/iterate/PhoenixQueryTimeoutIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/iterate/PhoenixQueryTimeoutIT.java @@ -29,14 +29,18 @@ import java.util.Properties; import java.util.Set; import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; +import org.apache.phoenix.end2end.BaseHBaseManagedTimeTableReuseIT; import org.apache.phoenix.jdbc.PhoenixStatement; +import org.junit.BeforeClass; import org.junit.Test; /** * Tests to validate that user specified property phoenix.query.timeoutMs * works as expected. */ -public class PhoenixQueryTimeoutIT extends BaseHBaseManagedTimeIT { +public class PhoenixQueryTimeoutIT extends BaseHBaseManagedTimeTableReuseIT { + + private static final String QUERY_TIMEOUT_TEST = generateRandomString(); @Test /** @@ -83,32 +87,31 @@ public class PhoenixQueryTimeoutIT extends BaseHBaseManagedTimeIT { //----------------------------------------------------------------- private PreparedStatement loadDataAndPrepareQuery(int timeoutMs, int timeoutSecs) throws Exception, SQLException { - createTableAndInsertRows(1000); Properties props = new Properties(); props.setProperty("phoenix.query.timeoutMs", String.valueOf(timeoutMs)); Connection conn = DriverManager.getConnection(getUrl(), props); - PreparedStatement ps = conn.prepareStatement("SELECT * FROM QUERY_TIMEOUT_TEST"); + PreparedStatement ps = conn.prepareStatement("SELECT * FROM " + QUERY_TIMEOUT_TEST); PhoenixStatement phoenixStmt = ps.unwrap(PhoenixStatement.class); assertEquals(timeoutMs, phoenixStmt.getQueryTimeoutInMillis()); assertEquals(timeoutSecs, phoenixStmt.getQueryTimeout()); return ps; } - - private Set<String> createTableAndInsertRows(int numRows) throws Exception { - String ddl = "CREATE TABLE QUERY_TIMEOUT_TEST (K VARCHAR NOT NULL PRIMARY KEY, V VARCHAR)"; + + @BeforeClass + public static void createTableAndInsertRows() throws Exception { + int numRows = 1000; + String ddl = + "CREATE TABLE " + QUERY_TIMEOUT_TEST + " (K VARCHAR NOT NULL PRIMARY KEY, V VARCHAR)"; Connection conn = DriverManager.getConnection(getUrl()); conn.createStatement().execute(ddl); - String dml = "UPSERT INTO QUERY_TIMEOUT_TEST VALUES (?, ?)"; + String dml = "UPSERT INTO " + QUERY_TIMEOUT_TEST + " VALUES (?, ?)"; PreparedStatement stmt = conn.prepareStatement(dml); - final Set<String> expectedKeys = new HashSet<>(numRows); for (int i = 1; i <= numRows; i++) { String key = "key" + i; - expectedKeys.add(key); stmt.setString(1, key); stmt.setString(2, "value" + i); stmt.executeUpdate(); } conn.commit(); - return expectedKeys; } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/ea3b8eaf/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java b/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java index 71d1b69..6dbd78d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/memory/GlobalMemoryManager.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.memory; +import com.google.common.annotations.VisibleForTesting; import org.apache.http.annotation.GuardedBy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,17 +73,7 @@ public class GlobalMemoryManager implements MemoryManager { long nBytes; synchronized(sync) { while (usedMemoryBytes + minBytes > maxMemoryBytes) { // Only wait if minBytes not available - try { - logger.debug("Waiting for " + (usedMemoryBytes + minBytes - maxMemoryBytes) + " bytes to be free"); - long remainingWaitTimeMs = maxWaitMs - (System.currentTimeMillis() - startTimeMs); - if (remainingWaitTimeMs <= 0) { // Ran out of time waiting for some memory to get freed up - throw new InsufficientMemoryException("Requested memory of " + minBytes + " bytes could not be allocated. Using memory of " + usedMemoryBytes + " bytes from global pool of " + maxMemoryBytes + " bytes after waiting for " + maxWaitMs + "ms."); - } - sync.wait(remainingWaitTimeMs); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Interrupted allocation of " + minBytes + " bytes", ie); - } + waitForBytesToFree(minBytes, startTimeMs); } // Allocate at most reqBytes, but at least minBytes nBytes = Math.min(reqBytes, maxMemoryBytes - usedMemoryBytes); @@ -94,6 +85,21 @@ public class GlobalMemoryManager implements MemoryManager { return nBytes; } + @VisibleForTesting + void waitForBytesToFree(long minBytes, long startTimeMs) { + try { + logger.debug("Waiting for " + (usedMemoryBytes + minBytes - maxMemoryBytes) + " bytes to be free " + startTimeMs); + long remainingWaitTimeMs = maxWaitMs - (System.currentTimeMillis() - startTimeMs); + if (remainingWaitTimeMs <= 0) { // Ran out of time waiting for some memory to get freed up + throw new InsufficientMemoryException("Requested memory of " + minBytes + " bytes could not be allocated. Using memory of " + usedMemoryBytes + " bytes from global pool of " + maxMemoryBytes + " bytes after waiting for " + maxWaitMs + "ms."); + } + sync.wait(remainingWaitTimeMs); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted allocation of " + minBytes + " bytes", ie); + } + } + @Override public MemoryChunk allocate(long minBytes, long reqBytes) { long nBytes = allocateBytes(minBytes, reqBytes); @@ -105,7 +111,7 @@ public class GlobalMemoryManager implements MemoryManager { return allocate(nBytes,nBytes); } - protected MemoryChunk newMemoryChunk(long sizeBytes) { + private MemoryChunk newMemoryChunk(long sizeBytes) { return new GlobalMemoryChunk(sizeBytes); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/ea3b8eaf/phoenix-core/src/test/java/org/apache/phoenix/memory/MemoryManagerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/memory/MemoryManagerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/memory/MemoryManagerTest.java index 94e6c79..e7aaa69 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/memory/MemoryManagerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/memory/MemoryManagerTest.java @@ -19,19 +19,24 @@ package org.apache.phoenix.memory; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.spy; -import org.junit.Ignore; import org.junit.Test; import org.apache.phoenix.memory.MemoryManager.MemoryChunk; +import org.mockito.Mockito; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; /** - * + * * Tests for GlobalMemoryManager and ChildMemoryManager * TODO: use our own time keeper so these tests don't flap * - * + * * @since 0.1 */ public class MemoryManagerTest { @@ -53,7 +58,7 @@ public class MemoryManagerTest { fail(); } catch (InsufficientMemoryException e) { // expected } - + c1.close(); c2.close(); assertTrue(rmm1.getAvailableMemory() == rmm1.getMaxMemory()); @@ -69,35 +74,37 @@ public class MemoryManagerTest { } @Test - public void testWaitForMemoryAvailable() { - final GlobalMemoryManager gmm = new GlobalMemoryManager(100,8000); + public void testWaitForMemoryAvailable() throws Exception { + final GlobalMemoryManager gmm = spy(new GlobalMemoryManager(100, 80)); final ChildMemoryManager rmm1 = new ChildMemoryManager(gmm,100); final ChildMemoryManager rmm2 = new ChildMemoryManager(gmm,100); + final CountDownLatch latch = new CountDownLatch(2); Thread t1 = new Thread() { @Override public void run() { MemoryChunk c1 = rmm1.allocate(50); MemoryChunk c2 = rmm1.allocate(50); - sleepFor(4000); + sleepFor(40); c1.close(); - sleepFor(2000); + sleepFor(20); c2.close(); + latch.countDown(); } }; Thread t2 = new Thread() { @Override public void run() { - sleepFor(2000); + sleepFor(20); // Will require waiting for a bit of time before t1 frees the requested memory - long startTime = System.currentTimeMillis(); MemoryChunk c3 = rmm2.allocate(50); - assertTrue(System.currentTimeMillis() - startTime >= 1000); + Mockito.verify(gmm, atLeastOnce()).waitForBytesToFree(anyLong(), anyLong()); c3.close(); + latch.countDown(); } }; - t1.start(); t2.start(); - sleepFor(1000); + t1.start(); + latch.await(1, TimeUnit.SECONDS); // Main thread competes with others to get all memory, but should wait // until both threads are complete (since that's when the memory will // again be all available. @@ -108,39 +115,40 @@ public class MemoryManagerTest { assertTrue(rmm1.getAvailableMemory() == rmm1.getMaxMemory()); assertTrue(rmm2.getAvailableMemory() == rmm2.getMaxMemory()); } - + @Test - @Ignore // TODO: get this not to flap - public void testResizeWaitForMemoryAvailable() { - final GlobalMemoryManager gmm = new GlobalMemoryManager(100,8000); + public void testResizeWaitForMemoryAvailable() throws Exception { + final GlobalMemoryManager gmm = spy(new GlobalMemoryManager(100, 80)); final ChildMemoryManager rmm1 = new ChildMemoryManager(gmm,100); final ChildMemoryManager rmm2 = new ChildMemoryManager(gmm,100); + final CountDownLatch latch = new CountDownLatch(2); Thread t1 = new Thread() { @Override public void run() { MemoryChunk c1 = rmm1.allocate(50); MemoryChunk c2 = rmm1.allocate(40); - sleepFor(4000); + sleepFor(40); c1.close(); - sleepFor(2000); + sleepFor(20); c2.close(); + latch.countDown(); } }; Thread t2 = new Thread() { @Override public void run() { - sleepFor(2000); + sleepFor(20); MemoryChunk c3 = rmm2.allocate(10); // Will require waiting for a bit of time before t1 frees the requested memory - long startTime = System.currentTimeMillis(); c3.resize(50); - assertTrue(System.currentTimeMillis() - startTime >= 2000); + Mockito.verify(gmm, atLeastOnce()).waitForBytesToFree(anyLong(), anyLong()); c3.close(); + latch.countDown(); } }; t1.start(); t2.start(); - sleepFor(3000); + latch.await(1, TimeUnit.SECONDS); // Main thread competes with others to get all memory, but should wait // until both threads are complete (since that's when the memory will // again be all available. @@ -151,41 +159,47 @@ public class MemoryManagerTest { assertTrue(rmm1.getAvailableMemory() == rmm1.getMaxMemory()); assertTrue(rmm2.getAvailableMemory() == rmm2.getMaxMemory()); } - - // @Test commenting out because the test is flapping too often - public void broken_testWaitUntilResize() { - final GlobalMemoryManager gmm = new GlobalMemoryManager(100,8000); + + @Test + public void testWaitUntilResize() throws Exception { + final GlobalMemoryManager gmm = spy(new GlobalMemoryManager(100, 80)); final ChildMemoryManager rmm1 = new ChildMemoryManager(gmm,100); final MemoryChunk c1 = rmm1.allocate(70); + final CountDownLatch latch = new CountDownLatch(2); + Thread t1 = new Thread() { @Override public void run() { MemoryChunk c2 = rmm1.allocate(20); - sleepFor(4000); + sleepFor(40); c1.resize(20); // resize down to test that other thread is notified - sleepFor(2000); + sleepFor(20); c2.close(); c1.close(); assertTrue(rmm1.getAvailableMemory() == rmm1.getMaxMemory()); + latch.countDown(); } }; Thread t2 = new Thread() { @Override public void run() { - sleepFor(2000); + sleepFor(20); ChildMemoryManager rmm2 = new ChildMemoryManager(gmm,100); MemoryChunk c3 = rmm2.allocate(10); long startTime = System.currentTimeMillis(); c3.resize(60); // Test that resize waits if memory not available assertTrue(c1.getSize() == 20); // c1 was resized not closed - assertTrue(System.currentTimeMillis() - startTime >= 2000); // we waited some time before the allocate happened + // we waited some time before the allocate happened + + Mockito.verify(gmm, atLeastOnce()).waitForBytesToFree(anyLong(), anyLong()); c3.close(); assertTrue(rmm2.getAvailableMemory() == rmm2.getMaxMemory()); + latch.countDown(); } }; t1.start(); t2.start(); - sleepFor(1000); + latch.await(1, TimeUnit.SECONDS); // Main thread competes with others to get all memory, but should wait // until both threads are complete (since that's when the memory will // again be all available. @@ -197,7 +211,7 @@ public class MemoryManagerTest { @Test public void testChildDecreaseAllocation() throws Exception { - MemoryManager gmm = new GlobalMemoryManager(100,1); + MemoryManager gmm = spy(new GlobalMemoryManager(100, 1)); ChildMemoryManager rmm1 = new ChildMemoryManager(gmm,100); ChildMemoryManager rmm2 = new ChildMemoryManager(gmm,10); MemoryChunk c1 = rmm1.allocate(50); @@ -209,7 +223,7 @@ public class MemoryManagerTest { assertTrue(rmm2.getAvailableMemory() == rmm2.getMaxMemory()); assertTrue(gmm.getAvailableMemory() == gmm.getMaxMemory()); } - + @Test public void testOverChildMemoryLimit() throws Exception { MemoryManager gmm = new GlobalMemoryManager(100,1); @@ -229,7 +243,7 @@ public class MemoryManagerTest { // Ensure that you can get back to max for rmn1 after failure MemoryChunk c4 = rmm1.allocate(10); MemoryChunk c5 = rmm1.allocate(15); - + MemoryChunk c6 = rmm4.allocate(25); try { // This passes % test, but fails the next total memory usage test @@ -241,7 +255,7 @@ public class MemoryManagerTest { // Tests that % test passes (confirming that the 10 above was subtracted back from request memory usage, // since we'd be at the max of 35% now MemoryChunk c7 = rmm4.allocate(10); - + try { rmm4.allocate(1); fail(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/ea3b8eaf/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java index 8b9b74f..8c3678d 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java @@ -118,6 +118,9 @@ import java.util.concurrent.TimeoutException; import javax.annotation.Nonnull; + +import org.apache.commons.lang.RandomStringUtils; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -125,6 +128,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.IntegrationTestingUtility; +import org.apache.hadoop.hbase.RandomStringGeneratorImpl; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; @@ -828,6 +832,10 @@ public abstract class BaseTest { createTestTable(url, ddl, splits, ts); } + protected static String generateRandomString() { + return RandomStringUtils.randomAlphabetic(20).toUpperCase(); + } + protected static void createTestTable(String url, String ddl) throws SQLException { createTestTable(url, ddl, null, null); }
