This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 5c2d211a88c1190a02ce7720fda7b0871f1e5cf5
Author: Bharath Krishna <bhar...@cloudera.com>
AuthorDate: Wed Apr 17 09:54:13 2019 -0700
IMPALA-8430: Fix flakiness in testCreateDropCreateDatabase
The test fails because of two Databases getting created with
same CREATION_TIME. Hence, adding a sleep of 2 seconds to
avoid this case. Also fixing other tests with similar use-case.
Testing
- Fixed the unit tests
Change-Id: I30bf4535d54c9cd8d257b528dc7a1b42f106800d
Reviewed-on: http://gerrit.cloudera.org:8080/13058
Reviewed-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com>
---
.../impala/catalog/events/MetastoreEvents.java | 23 ++++++++++++++++++++--
.../events/MetastoreEventsProcessorTest.java | 22 ++++++++++++++++++---
2 files changed, 40 insertions(+), 5 deletions(-)
diff --git
a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
index 4f2b6e7..8b5b6f3 100644
--- a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
+++ b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
@@ -777,7 +777,16 @@ public class MetastoreEvents {
* catalog, ignore the event. If the table exists in the catalog, compares
the
* createTime of the table in catalog with the createTime of the table
from the event
* and remove the catalog table if there is a match. If the catalog table
is a
- * incomplete table it is removed as well.
+ * incomplete table it is removed as well. The creation_time from HMS is
unfortunately
+ * in seconds granularity, which means there is a limitation that we cannot
+ * distinguish between tables which are created with the same name within
a second.
+ * So a sequence of create_table, drop_table, create_table happening
within the
+ * same second might cause false positives on drop_table event processing.
This is
+ * not a huge problem since the tables will eventually be created when the
+ * create events are processed but there will be a non-zero amount of time
when the
+ * table will not be existing in catalog.
+ * TODO : Once HIVE-21595 is available we should rely on table_id for
determining a
+ * newer incarnation of a previous table.
*/
@Override
public void process() {
@@ -900,7 +909,17 @@ public class MetastoreEvents {
* CREATION_TIME of the catalog's DB object is greater than that of the
notification
* event's DB object, it means that the Database object present in the
catalog is a
* later version and we can skip the event. (For instance, when user does
a create db,
- * drop db and create db again with the same dbName.)
+ * drop db and create db again with the same dbName.).
+ * The creation_time from HMS is unfortunately in seconds granularity,
which means
+ * there is a limitation that we cannot distinguish between databases
which are
+ * created with the same name within a second. So a sequence of
create_database,
+ * drop_database, create_database happening within the same second might
cause
+ * false positives on drop_database event processing. This is not a huge
problem
+ * since the databases will eventually be created when the create events
are
+ * processed but there will be a non-zero amount of time when the database
will not
+ * be existing in catalog.
+ * TODO : Once HIVE-21595 is available we should rely on database_id for
determining a
+ * newer incarnation of a previous database.
*/
@Override
public void process() {
diff --git
a/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
b/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
index b9982e9..00d8b78 100644
---
a/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
+++
b/fe/src/test/java/org/apache/impala/catalog/events/MetastoreEventsProcessorTest.java
@@ -300,7 +300,7 @@ public class MetastoreEventsProcessorTest {
// as the previous CREATE_DB operation, so as to trigger the filtering
logic
// based on CREATION_TIME in DROP_DB event processing. This is currently a
// limitation : the DROP_DB event filtering expects that while processing
events,
- // the CREATION_TIME of two DB's with same name won't have the same
+ // the CREATION_TIME of two Databases with same name won't have the same
// creation timestamp.
sleep(2000);
// Create database again with same name
@@ -687,7 +687,8 @@ public class MetastoreEventsProcessorTest {
* already at its latest state
*/
@Test
- public void testCreateDropCreateTableFromImpala() throws ImpalaException,
TException {
+ public void testCreateDropCreateTableFromImpala()
+ throws ImpalaException, TException, InterruptedException {
assertEquals(EventProcessorStatus.ACTIVE, eventsProcessor_.getStatus());
createDatabase(TEST_DB_NAME, null);
final String testTblName = "testCreateDropCreateTableFromImpala";
@@ -696,6 +697,13 @@ public class MetastoreEventsProcessorTest {
assertNotNull("Table should have been found after create table statement",
catalog_.getTable(TEST_DB_NAME, testTblName));
loadTable(testTblName);
+ // Adding sleep here to make sure that the CREATION_TIME is not same
+ // as the previous CREATE_TABLE operation, so as to trigger the filtering
logic
+ // based on CREATION_TIME in DROP_TABLE event processing. This is
currently a
+ // limitation : the DROP_TABLE event filtering expects that while
processing events,
+ // the CREATION_TIME of two tables with same name won't have the same
+ // creation timestamp.
+ sleep(2000);
dropTableFromImpala(TEST_DB_NAME, testTblName);
// now catalogD does not have the table entry, create the table again
createTableFromImpala(TEST_DB_NAME, testTblName, false);
@@ -841,11 +849,19 @@ public class MetastoreEventsProcessorTest {
* of Table.
*/
@Test
- public void testCreateDropCreateDatabaseFromImpala() throws ImpalaException {
+ public void testCreateDropCreateDatabaseFromImpala()
+ throws ImpalaException, InterruptedException {
assertEquals(EventProcessorStatus.ACTIVE, eventsProcessor_.getStatus());
createDatabaseFromImpala(TEST_DB_NAME, "first");
assertNotNull("Db should have been found after create database statement",
catalog_.getDb(TEST_DB_NAME));
+ // Adding sleep here to make sure that the CREATION_TIME is not same
+ // as the previous CREATE_DB operation, so as to trigger the filtering
logic
+ // based on CREATION_TIME in DROP_DB event processing. This is currently a
+ // limitation : the DROP_DB event filtering expects that while processing
events,
+ // the CREATION_TIME of two Databases with same name won't have the same
+ // creation timestamp.
+ sleep(2000);
dropDatabaseCascadeFromImpala(TEST_DB_NAME);
assertNull(catalog_.getDb(TEST_DB_NAME));
createDatabaseFromImpala(TEST_DB_NAME, "second");
