Repository: hive Updated Branches: refs/heads/master 1e437e2b1 -> 369490860
HIVE-20226: HMS getNextNotification will throw exception when request maxEvents exceed table's max_rows (Alice Fan, reviewed by Yongzhi Chen) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/36949086 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/36949086 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/36949086 Branch: refs/heads/master Commit: 3694908602c3182eea1171f2696fae1facc245fb Parents: 1e437e2 Author: Yongzhi Chen <[email protected]> Authored: Fri Jul 27 10:05:37 2018 -0400 Committer: Yongzhi Chen <[email protected]> Committed: Fri Jul 27 10:10:22 2018 -0400 ---------------------------------------------------------------------- .../apache/hadoop/hive/metastore/ObjectStore.java | 3 ++- .../hadoop/hive/metastore/conf/MetastoreConf.java | 2 ++ .../hadoop/hive/metastore/TestObjectStore.java | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/36949086/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 606ae50..e6f9acb 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -9795,10 +9795,11 @@ public class ObjectStore implements RawStore, Configurable { try { openTransaction(); long lastEvent = rqst.getLastEvent(); - int maxEvents = rqst.getMaxEvents() > 0 ? rqst.getMaxEvents() : Integer.MAX_VALUE; query = pm.newQuery(MNotificationLog.class, "eventId > lastEvent"); query.declareParameters("java.lang.Long lastEvent"); query.setOrdering("eventId ascending"); + int maxEventResponse = MetastoreConf.getIntVar(conf, ConfVars.METASTORE_MAX_EVENT_RESPONSE); + int maxEvents = (rqst.getMaxEvents() < maxEventResponse && rqst.getMaxEvents() > 0) ? rqst.getMaxEvents() : maxEventResponse; query.setRange(0, maxEvents); Collection<MNotificationLog> events = (Collection) query.execute(lastEvent); commited = commitTransaction(); http://git-wip-us.apache.org/repos/asf/hive/blob/36949086/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 5791375..cf5fbbe 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -594,6 +594,8 @@ public class MetastoreConf { "When used in conjunction with the org.apache.hadoop.hive.ql.parse.MetaDataExportListener pre event listener, \n" + "it is the location to which the metadata will be exported. The default is an empty string, which results in the \n" + "metadata being exported to the current user's home directory on HDFS."), + METASTORE_MAX_EVENT_RESPONSE("metastore.max.event.response", "hive.metastore.max.event.response", 1000000, + "The parameter will decide the maximum number of events that HMS will respond."), MOVE_EXPORTED_METADATA_TO_TRASH("metastore.metadata.move.exported.metadata.to.trash", "hive.metadata.move.exported.metadata.to.trash", true, "When used in conjunction with the org.apache.hadoop.hive.ql.parse.MetaDataExportListener pre event listener, \n" + http://git-wip-us.apache.org/repos/asf/hive/blob/36949086/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java index e53ad77..b74c304 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -811,6 +811,24 @@ public class TestObjectStore { Assert.assertEquals(0, eventResponse.getEventsSize()); } + /** + * Test metastore configuration property METASTORE_MAX_EVENT_RESPONSE + */ + @Test + public void testMaxEventResponse() throws InterruptedException, MetaException { + NotificationEvent event = + new NotificationEvent(0, 0, EventMessage.EventType.CREATE_DATABASE.toString(), ""); + MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.METASTORE_MAX_EVENT_RESPONSE, 1); + ObjectStore objs = new ObjectStore(); + objs.setConf(conf); + // Verify if METASTORE_MAX_EVENT_RESPONSE will limit number of events to respond + for (int i = 0; i < 3; i++) { + objs.addNotificationEvent(event); + } + NotificationEventResponse eventResponse = objs.getNextNotification(new NotificationEventRequest()); + Assert.assertEquals(1, eventResponse.getEventsSize()); + } + @Ignore( "This test is here to allow testing with other databases like mysql / postgres etc\n" + " with user changes to the code. This cannot be run on apache derby because of\n"
