HIVE-15705: Event replication for constraints (Daniel Dai, reviewed by Sankar 
Hariappan, Thejas Nair)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/90a92b74
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/90a92b74
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/90a92b74

Branch: refs/heads/master
Commit: 90a92b7462f17bddcc55a02736a0af884db2d018
Parents: 16bfb9c
Author: Daniel Dai <da...@hortonworks.com>
Authored: Wed Aug 9 12:35:49 2017 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Wed Aug 9 12:35:49 2017 -0700

----------------------------------------------------------------------
 .../listener/DbNotificationListener.java        |   94 +
 .../listener/DummyRawStoreFailEvent.java        |   15 +-
 .../hive/ql/parse/TestReplicationScenarios.java | 2342 +++++++++---------
 .../hadoop/hive/metastore/HiveMetaStore.java    |  230 +-
 .../hive/metastore/MetaStoreEventListener.java  |   40 +
 .../metastore/MetaStoreListenerNotifier.java    |   28 +
 .../hadoop/hive/metastore/ObjectStore.java      |   66 +-
 .../apache/hadoop/hive/metastore/RawStore.java  |   10 +-
 .../hive/metastore/cache/CachedStore.java       |   21 +-
 .../metastore/events/AddForeignKeyEvent.java    |   37 +
 .../events/AddNotNullConstraintEvent.java       |   37 +
 .../metastore/events/AddPrimaryKeyEvent.java    |   38 +
 .../events/AddUniqueConstraintEvent.java        |   37 +
 .../metastore/events/DropConstraintEvent.java   |   47 +
 .../messaging/AddForeignKeyMessage.java         |   36 +
 .../messaging/AddNotNullConstraintMessage.java  |   36 +
 .../messaging/AddPrimaryKeyMessage.java         |   35 +
 .../messaging/AddUniqueConstraintMessage.java   |   36 +
 .../messaging/DropConstraintMessage.java        |   29 +
 .../hive/metastore/messaging/EventMessage.java  |    7 +-
 .../messaging/MessageDeserializer.java          |   36 +-
 .../metastore/messaging/MessageFactory.java     |   53 +-
 .../json/JSONAddForeignKeyMessage.java          |  101 +
 .../json/JSONAddNotNullConstraintMessage.java   |   79 +
 .../json/JSONAddPrimaryKeyMessage.java          |  101 +
 .../json/JSONAddUniqueConstraintMessage.java    |   98 +
 .../json/JSONDropConstraintMessage.java         |   90 +
 .../messaging/json/JSONMessageDeserializer.java |   50 +
 .../messaging/json/JSONMessageFactory.java      |   56 +
 .../DummyRawStoreControlledCommit.java          |   17 +-
 .../DummyRawStoreForJdoConnection.java          |   15 +-
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |   13 +-
 .../hive/ql/parse/DDLSemanticAnalyzer.java      |    4 +-
 .../hadoop/hive/ql/parse/repl/DumpType.java     |   35 +
 .../repl/dump/events/AddForeignKeyHandler.java  |   41 +
 .../events/AddNotNullConstraintHandler.java     |   41 +
 .../repl/dump/events/AddPrimaryKeyHandler.java  |   41 +
 .../dump/events/AddUniqueConstraintHandler.java |   41 +
 .../repl/dump/events/DropConstraintHandler.java |   41 +
 .../repl/dump/events/EventHandlerFactory.java   |    5 +
 .../hive/ql/parse/repl/load/DumpMetaData.java   |   14 +-
 .../repl/load/message/AddForeignKeyHandler.java |   71 +
 .../message/AddNotNullConstraintHandler.java    |   72 +
 .../repl/load/message/AddPrimaryKeyHandler.java |   69 +
 .../message/AddUniqueConstraintHandler.java     |   70 +
 .../load/message/DropConstraintHandler.java     |   51 +
 .../hadoop/hive/ql/plan/AlterTableDesc.java     |   19 +-
 47 files changed, 3352 insertions(+), 1193 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
----------------------------------------------------------------------
diff --git 
a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
 
b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
index 351546c..2a2cc7b 100644
--- 
a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
+++ 
b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java
@@ -41,9 +41,17 @@ import org.apache.hadoop.hive.metastore.api.Index;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.NotificationEvent;
 import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
+import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
+import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
+import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
 import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.events.AddForeignKeyEvent;
 import org.apache.hadoop.hive.metastore.events.AddIndexEvent;
+import org.apache.hadoop.hive.metastore.events.AddNotNullConstraintEvent;
 import org.apache.hadoop.hive.metastore.events.AddPartitionEvent;
+import org.apache.hadoop.hive.metastore.events.AddPrimaryKeyEvent;
+import org.apache.hadoop.hive.metastore.events.AddUniqueConstraintEvent;
 import org.apache.hadoop.hive.metastore.events.AlterIndexEvent;
 import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
 import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
@@ -51,6 +59,7 @@ import 
org.apache.hadoop.hive.metastore.events.ConfigChangeEvent;
 import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
 import org.apache.hadoop.hive.metastore.events.CreateFunctionEvent;
 import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
+import org.apache.hadoop.hive.metastore.events.DropConstraintEvent;
 import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
 import org.apache.hadoop.hive.metastore.events.DropFunctionEvent;
 import org.apache.hadoop.hive.metastore.events.DropIndexEvent;
@@ -459,6 +468,91 @@ public class DbNotificationListener extends 
MetaStoreEventListener {
     // then load data into it.
   }
 
+  /***
+   * @param addPrimaryKeyEvent add primary key event
+   * @throws MetaException
+   */
+  @Override
+  public void onAddPrimaryKey(AddPrimaryKeyEvent addPrimaryKeyEvent) throws 
MetaException {
+    List<SQLPrimaryKey> cols = addPrimaryKeyEvent.getPrimaryKeyCols();
+    if (cols.size() > 0) {
+      NotificationEvent event =
+          new NotificationEvent(0, now(), EventType.ADD_PRIMARYKEY.toString(), 
msgFactory
+              
.buildAddPrimaryKeyMessage(addPrimaryKeyEvent.getPrimaryKeyCols()).toString());
+      event.setDbName(cols.get(0).getTable_db());
+      event.setTableName(cols.get(0).getTable_name());
+      process(event, addPrimaryKeyEvent);
+    }
+  }
+
+  /***
+   * @param addForeignKeyEvent add foreign key event
+   * @throws MetaException
+   */
+  @Override
+  public void onAddForeignKey(AddForeignKeyEvent addForeignKeyEvent) throws 
MetaException {
+    List<SQLForeignKey> cols = addForeignKeyEvent.getForeignKeyCols();
+    if (cols.size() > 0) {
+      NotificationEvent event =
+          new NotificationEvent(0, now(), EventType.ADD_FOREIGNKEY.toString(), 
msgFactory
+              
.buildAddForeignKeyMessage(addForeignKeyEvent.getForeignKeyCols()).toString());
+      event.setDbName(cols.get(0).getPktable_db());
+      event.setTableName(cols.get(0).getPktable_name());
+      process(event, addForeignKeyEvent);
+    }
+  }
+
+  /***
+   * @param addUniqueConstraintEvent add unique constraint event
+   * @throws MetaException
+   */
+  @Override
+  public void onAddUniqueConstraint(AddUniqueConstraintEvent 
addUniqueConstraintEvent) throws MetaException {
+    List<SQLUniqueConstraint> cols = 
addUniqueConstraintEvent.getUniqueConstraintCols();
+    if (cols.size() > 0) {
+      NotificationEvent event =
+          new NotificationEvent(0, now(), 
EventType.ADD_UNIQUECONSTRAINT.toString(), msgFactory
+              
.buildAddUniqueConstraintMessage(addUniqueConstraintEvent.getUniqueConstraintCols()).toString());
+      event.setDbName(cols.get(0).getTable_db());
+      event.setTableName(cols.get(0).getTable_name());
+      process(event, addUniqueConstraintEvent);
+    }
+  }
+
+  /***
+   * @param addNotNullConstraintEvent add not null constraint event
+   * @throws MetaException
+   */
+  @Override
+  public void onAddNotNullConstraint(AddNotNullConstraintEvent 
addNotNullConstraintEvent) throws MetaException {
+    List<SQLNotNullConstraint> cols = 
addNotNullConstraintEvent.getNotNullConstraintCols();
+    if (cols.size() > 0) {
+      NotificationEvent event =
+          new NotificationEvent(0, now(), 
EventType.ADD_NOTNULLCONSTRAINT.toString(), msgFactory
+              
.buildAddNotNullConstraintMessage(addNotNullConstraintEvent.getNotNullConstraintCols()).toString());
+      event.setDbName(cols.get(0).getTable_db());
+      event.setTableName(cols.get(0).getTable_name());
+      process(event, addNotNullConstraintEvent);
+    }
+  }
+
+  /***
+   * @param dropConstraintEvent drop constraint event
+   * @throws MetaException
+   */
+  @Override
+  public void onDropConstraint(DropConstraintEvent dropConstraintEvent) throws 
MetaException {
+    String dbName = dropConstraintEvent.getDbName();
+    String tableName = dropConstraintEvent.getTableName();
+    String constraintName = dropConstraintEvent.getConstraintName();
+    NotificationEvent event =
+        new NotificationEvent(0, now(), EventType.DROP_CONSTRAINT.toString(), 
msgFactory
+            .buildDropConstraintMessage(dbName, tableName, 
constraintName).toString());
+    event.setDbName(dbName);
+    event.setTableName(tableName);
+    process(event, dropConstraintEvent);
+  }
+
   private int now() {
     long millis = System.currentTimeMillis();
     millis /= 1000;

http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
----------------------------------------------------------------------
diff --git 
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
 
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
index 35989f5..5d7cfad 100644
--- 
a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
+++ 
b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
@@ -912,11 +912,12 @@ public class DummyRawStoreFailEvent implements RawStore, 
Configurable {
   }
 
   @Override
-  public void createTableWithConstraints(Table tbl,
+  public List<String> createTableWithConstraints(Table tbl,
                                          List<SQLPrimaryKey> primaryKeys, 
List<SQLForeignKey> foreignKeys,
                                          List<SQLUniqueConstraint> 
uniqueConstraints,
                                          List<SQLNotNullConstraint> 
notNullConstraints)
       throws InvalidObjectException, MetaException {
+    return null;
   }
 
   @Override
@@ -925,23 +926,27 @@ public class DummyRawStoreFailEvent implements RawStore, 
Configurable {
   }
 
   @Override
-  public void addPrimaryKeys(List<SQLPrimaryKey> pks)
+  public List<String> addPrimaryKeys(List<SQLPrimaryKey> pks)
       throws InvalidObjectException, MetaException {
+    return null;
   }
 
   @Override
-  public void addForeignKeys(List<SQLForeignKey> fks)
+  public List<String> addForeignKeys(List<SQLForeignKey> fks)
       throws InvalidObjectException, MetaException {
+    return null;
   }
 
   @Override
-  public void addUniqueConstraints(List<SQLUniqueConstraint> uks)
+  public List<String> addUniqueConstraints(List<SQLUniqueConstraint> uks)
       throws InvalidObjectException, MetaException {
+    return null;
   }
 
   @Override
-  public void addNotNullConstraints(List<SQLNotNullConstraint> nns)
+  public List<String> addNotNullConstraints(List<SQLNotNullConstraint> nns)
       throws InvalidObjectException, MetaException {
+    return null;
   }
 
   @Override

Reply via email to