hive git commit: HIVE-20701: Allow HiveStreaming to receive a key value to commit atomically together with the transaction (Jaume M reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/master cbe3228c2 -> 7765e90aa
HIVE-20701: Allow HiveStreaming to receive a key value to commit atomically
together with the transaction (Jaume M reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7765e90a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7765e90a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7765e90a
Branch: refs/heads/master
Commit: 7765e90aad44747860b3c1adbe8a4857d864912d
Parents: cbe3228
Author: Jaume Marhuenda
Authored: Mon Oct 22 14:18:20 2018 -0700
Committer: Prasanth Jayachandran
Committed: Mon Oct 22 14:18:49 2018 -0700
--
.../streaming/AbstractStreamingTransaction.java | 6 ++-
.../hive/streaming/HiveStreamingConnection.java | 13 +--
.../hive/streaming/StreamingConnection.java | 23 ---
.../hive/streaming/StreamingTransaction.java| 14 ++-
.../apache/hive/streaming/TransactionBatch.java | 26 +++--
.../streaming/UnManagedSingleTransaction.java | 3 +-
.../apache/hive/streaming/TestStreaming.java| 41 +++-
7 files changed, 109 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/7765e90a/streaming/src/java/org/apache/hive/streaming/AbstractStreamingTransaction.java
--
diff --git
a/streaming/src/java/org/apache/hive/streaming/AbstractStreamingTransaction.java
b/streaming/src/java/org/apache/hive/streaming/AbstractStreamingTransaction.java
index a99fdba..6ab3ffe 100644
---
a/streaming/src/java/org/apache/hive/streaming/AbstractStreamingTransaction.java
+++
b/streaming/src/java/org/apache/hive/streaming/AbstractStreamingTransaction.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hive.metastore.api.TxnToWriteId;
import java.io.InputStream;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
/**
@@ -151,6 +152,9 @@ abstract class AbstractStreamingTransaction
}
public void commit() throws StreamingException {
-commitWithPartitions(null);
+commit(null);
+ }
+ public void commit(Set partitions) throws StreamingException {
+commit(partitions, null, null);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/7765e90a/streaming/src/java/org/apache/hive/streaming/HiveStreamingConnection.java
--
diff --git
a/streaming/src/java/org/apache/hive/streaming/HiveStreamingConnection.java
b/streaming/src/java/org/apache/hive/streaming/HiveStreamingConnection.java
index f79b844..74fc531 100644
--- a/streaming/src/java/org/apache/hive/streaming/HiveStreamingConnection.java
+++ b/streaming/src/java/org/apache/hive/streaming/HiveStreamingConnection.java
@@ -146,6 +146,7 @@ public class HiveStreamingConnection implements
StreamingConnection {
private boolean manageTransactions;
private int countTransactions = 0;
private Set partitions;
+ private Long tableId;
private HiveStreamingConnection(Builder builder) throws StreamingException {
this.database = builder.database.toLowerCase();
@@ -574,12 +575,18 @@ public class HiveStreamingConnection implements
StreamingConnection {
@Override
public void commitTransaction() throws StreamingException {
-commitTransactionWithPartition(null);
+commitTransaction(null);
}
@Override
- public void commitTransactionWithPartition(Set partitions)
+ public void commitTransaction(Set partitions)
throws StreamingException {
+commitTransaction(partitions, null, null);
+ }
+
+ @Override
+ public void commitTransaction(Set partitions, String key,
+ String value) throws StreamingException {
checkState();
Set createdPartitions = new HashSet<>();
@@ -598,7 +605,7 @@ public class HiveStreamingConnection implements
StreamingConnection {
connectionStats.incrementTotalPartitions(partitions.size());
}
-currentTransactionBatch.commitWithPartitions(createdPartitions);
+currentTransactionBatch.commit(createdPartitions, key, value);
this.partitions.addAll(
currentTransactionBatch.getPartitions());
connectionStats.incrementCreatedPartitions(createdPartitions.size());
http://git-wip-us.apache.org/repos/asf/hive/blob/7765e90a/streaming/src/java/org/apache/hive/streaming/StreamingConnection.java
--
diff --git
a/streaming/src/java/org/apache/hive/streaming/StreamingConnection.java
b/streaming/src/java/org/apache/hive/streaming/StreamingConnection.java
index 92016e5..ba4c6a5 100644
--- a/streaming/src/java/org/apache/hive/streaming/StreamingConnection.java
+++
hive git commit: HIVE-20765 : fetch partitions for txn stats validation in get_aggr_stats with one call (Sergey Shelukhin, reviewed by Ashutosh Chauhan, and Vihang Karajgaonkar)
Repository: hive
Updated Branches:
refs/heads/master b4302bb7a -> cbe3228c2
HIVE-20765 : fetch partitions for txn stats validation in get_aggr_stats with
one call (Sergey Shelukhin, reviewed by Ashutosh Chauhan, and Vihang
Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cbe3228c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cbe3228c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cbe3228c
Branch: refs/heads/master
Commit: cbe3228c29e2db6ae5cffcb94cdcf79063859ffe
Parents: b4302bb
Author: sergey
Authored: Mon Oct 22 14:06:48 2018 -0700
Committer: sergey
Committed: Mon Oct 22 14:06:48 2018 -0700
--
.../hadoop/hive/metastore/ObjectStore.java | 22 +---
.../metastore/PartitionProjectionEvaluator.java | 8 +--
2 files changed, 20 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/cbe3228c/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
--
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index b98b4b4..ddd64e7 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -9052,21 +9052,27 @@ public class ObjectStore implements RawStore,
Configurable {
return null;
}
- MTable table = getMTable(catName, dbName, tblName);
+ Table table = getTable(catName, dbName, tblName);
boolean isTxn = TxnUtils.isTransactionalTable(table.getParameters());
if (isTxn && !areTxnStatsSupported) {
return null;
}
+ GetPartitionsFilterSpec fs = new GetPartitionsFilterSpec();
+ fs.setFilterMode(PartitionFilterMode.BY_NAMES);
+ fs.setFilters(partNames);
+ GetPartitionsProjectionSpec ps = new GetPartitionsProjectionSpec();
+ ps.setIncludeParamKeyPattern(StatsSetupConst.COLUMN_STATS_ACCURATE +
'%');
+ ps.setFieldList(Lists.newArrayList("writeId", "parameters", "values"));
+ List parts = getPartitionSpecsByFilterAndProjection(table,
ps, fs);
// Loop through the given "partNames" list
// checking isolation-level-compliance of each partition column stats.
- for (String partName : partNames) {
-MPartition mpart = getMPartition(
-catName, dbName, tblName,
Warehouse.getPartValuesFromPartName(partName));
-if (!isCurrentStatsValidForTheQuery(mpart, writeIdList, false)) {
- LOG.debug("The current metastore transactional partition column
statistics " +
- "for " + dbName + "." + tblName + "." +
mpart.getPartitionName() + " is not valid " +
- "for the current query.");
+ for (Partition part : parts) {
+
+if (!isCurrentStatsValidForTheQuery(part, part.getWriteId(),
writeIdList, false)) {
+ String partName = Warehouse.makePartName(table.getPartitionKeys(),
part.getValues());
+ LOG.debug("The current metastore transactional partition column
statistics for " + dbName
+ + "." + tblName + "." + partName + " is not valid for the
current query");
return null;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/cbe3228c/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java
--
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java
index e918a33..c3a0093 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java
@@ -26,6 +26,7 @@ import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -38,6 +39,7 @@ import org.slf4j.LoggerFactory;
import javax.jdo.PersistenceManager;
import javax.jdo.Query;
+
import
[3/4] hive git commit: HIVE-20679: DDL operations on hive might create large messages for DBNotification (Anishek Agarwal, reviewed by Sankar Hariappan)
http://git-wip-us.apache.org/repos/asf/hive/blob/b4302bb7/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java
--
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java
index 3fe8b58..314ca48 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java
@@ -17,32 +17,19 @@
*/
package org.apache.hadoop.hive.ql.parse;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsRequest;
-import org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsResponse;
-import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
-import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
-import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
-import org.apache.hadoop.hive.metastore.txn.TxnStore;
-import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import
org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder;
import org.apache.hadoop.hive.shims.Utils;
-import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore;
-import
org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.CallerArguments;
-import
org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection;
+
import static
org.apache.hadoop.hive.metastore.ReplChangeManager.SOURCE_OF_REPLICATION;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.ErrorMsg;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+
import org.junit.rules.TestName;
-import org.junit.rules.TestRule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.junit.After;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -50,13 +37,11 @@ import org.junit.BeforeClass;
import org.junit.AfterClass;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
-import javax.annotation.Nullable;
-import java.util.Collections;
+import java.util.Map;
+
import com.google.common.collect.Lists;
-import org.junit.Ignore;
/**
* TestReplicationScenariosAcidTables - test replication for ACID tables
@@ -65,11 +50,9 @@ public class
TestReplicationScenariosIncrementalLoadAcidTables {
@Rule
public final TestName testName = new TestName();
- @Rule
- public TestRule replV1BackwardCompat;
-
protected static final Logger LOG =
LoggerFactory.getLogger(TestReplicationScenariosIncrementalLoadAcidTables.class);
- private static WarehouseInstance primary, replica, replicaNonAcid;
+ static WarehouseInstance primary;
+ private static WarehouseInstance replica, replicaNonAcid;
private static HiveConf conf;
private String primaryDbName, replicatedDbName, primaryDbNameExtra;
private enum OperationType {
@@ -80,12 +63,21 @@ public class
TestReplicationScenariosIncrementalLoadAcidTables {
@BeforeClass
public static void classLevelSetup() throws Exception {
-conf = new HiveConf(TestReplicationScenariosAcidTables.class);
+HashMap overrides = new HashMap<>();
+overrides.put(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(),
+GzipJSONMessageEncoder.class.getCanonicalName());
+
+internalBeforeClassSetup(overrides,
TestReplicationScenariosAcidTables.class);
+ }
+
+ static void internalBeforeClassSetup(Map overrides, Class
clazz)
+ throws Exception {
+conf = new HiveConf(clazz);
conf.set("dfs.client.use.datanode.hostname", "true");
conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() +
".hosts", "*");
MiniDFSCluster miniDFSCluster =
new
MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
-HashMap overridesForHiveConf = new HashMap() {{
+HashMap acidConfs = new HashMap() {{
put("fs.defaultFS",
miniDFSCluster.getFileSystem().getUri().toString());
put("hive.support.concurrency", "true");
put("hive.txn.manager",
"org.apache.hadoop.hive.ql.lockmgr.DbTxnManager");
@@ -97,9 +89,11 @@ public class
TestReplicationScenariosIncrementalLoadAcidTables {
put("mapred.input.dir.recursive", "true");
put("hive.metastore.disallow.incompatible.col.type.changes", "false");
}};
-primary = new WarehouseInstance(LOG, miniDFSCluster,
[2/4] hive git commit: HIVE-20679: DDL operations on hive might create large messages for DBNotification (Anishek Agarwal, reviewed by Sankar Hariappan)
http://git-wip-us.apache.org/repos/asf/hive/blob/b4302bb7/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java -- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java new file mode 100644 index 000..787b9b2 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java @@ -0,0 +1,425 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; +import java.util.regex.PatternSyntaxException; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.Catalog; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Function; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.SQLForeignKey; +import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; +import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; +import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.TxnToWriteId; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.events.AcidWriteEvent; +import org.apache.hadoop.hive.metastore.messaging.json.JSONAbortTxnMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONAcidWriteMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONAddForeignKeyMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONAddNotNullConstraintMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONAddPartitionMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONAddPrimaryKeyMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONAddUniqueConstraintMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONAllocWriteIdMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONAlterCatalogMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONAlterDatabaseMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONAlterPartitionMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONAlterTableMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONCommitTxnMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONCreateCatalogMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONCreateDatabaseMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONCreateFunctionMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONCreateTableMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONDropCatalogMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONDropConstraintMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONDropDatabaseMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONDropFunctionMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONDropPartitionMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONDropTableMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONInsertMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONOpenTxnMessage; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.thrift.TBase; +import
[1/4] hive git commit: HIVE-20679: DDL operations on hive might create large messages for DBNotification (Anishek Agarwal, reviewed by Sankar Hariappan)
Repository: hive Updated Branches: refs/heads/master b82995517 -> b4302bb7a http://git-wip-us.apache.org/repos/asf/hive/blob/b4302bb7/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java -- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java deleted file mode 100644 index 6aa079d..000 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java +++ /dev/null @@ -1,432 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.messaging.json; - -import java.util.Arrays; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Predicate; -import java.util.regex.PatternSyntaxException; - -import javax.annotation.Nullable; - -import org.apache.hadoop.hive.metastore.api.Catalog; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.Function; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.NotificationEvent; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.SQLForeignKey; -import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; -import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; -import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.api.TxnToWriteId; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.events.AcidWriteEvent; -import org.apache.hadoop.hive.metastore.messaging.AbortTxnMessage; -import org.apache.hadoop.hive.metastore.messaging.AddForeignKeyMessage; -import org.apache.hadoop.hive.metastore.messaging.AddNotNullConstraintMessage; -import org.apache.hadoop.hive.metastore.messaging.AddPartitionMessage; -import org.apache.hadoop.hive.metastore.messaging.AddPrimaryKeyMessage; -import org.apache.hadoop.hive.metastore.messaging.AddUniqueConstraintMessage; -import org.apache.hadoop.hive.metastore.messaging.AllocWriteIdMessage; -import org.apache.hadoop.hive.metastore.messaging.AlterCatalogMessage; -import org.apache.hadoop.hive.metastore.messaging.AlterDatabaseMessage; -import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage; -import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage; -import org.apache.hadoop.hive.metastore.messaging.CommitTxnMessage; -import org.apache.hadoop.hive.metastore.messaging.CreateCatalogMessage; -import org.apache.hadoop.hive.metastore.messaging.CreateDatabaseMessage; -import org.apache.hadoop.hive.metastore.messaging.CreateFunctionMessage; -import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage; -import org.apache.hadoop.hive.metastore.messaging.DropCatalogMessage; -import org.apache.hadoop.hive.metastore.messaging.DropConstraintMessage; -import org.apache.hadoop.hive.metastore.messaging.DropDatabaseMessage; -import org.apache.hadoop.hive.metastore.messaging.DropFunctionMessage; -import org.apache.hadoop.hive.metastore.messaging.DropPartitionMessage; -import org.apache.hadoop.hive.metastore.messaging.DropTableMessage; -import org.apache.hadoop.hive.metastore.messaging.InsertMessage; -import org.apache.hadoop.hive.metastore.messaging.MessageDeserializer; -import org.apache.hadoop.hive.metastore.messaging.MessageFactory; -import org.apache.hadoop.hive.metastore.messaging.OpenTxnMessage; -import org.apache.hadoop.hive.metastore.messaging.AcidWriteMessage; -import org.apache.hadoop.hive.metastore.messaging.PartitionFiles; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; -import org.apache.thrift.TBase; -import org.apache.thrift.TDeserializer; -import org.apache.thrift.TException; -import
[4/4] hive git commit: HIVE-20679: DDL operations on hive might create large messages for DBNotification (Anishek Agarwal, reviewed by Sankar Hariappan)
HIVE-20679: DDL operations on hive might create large messages for DBNotification (Anishek Agarwal, reviewed by Sankar Hariappan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b4302bb7 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b4302bb7 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b4302bb7 Branch: refs/heads/master Commit: b4302bb7ad967f15ca1b708685b2ac669e3cf037 Parents: b829955 Author: Anishek Agarwal Authored: Mon Oct 22 13:51:43 2018 +0530 Committer: Anishek Agarwal Committed: Mon Oct 22 13:51:43 2018 +0530 -- .../org/apache/hadoop/hive/conf/HiveConf.java | 2 +- .../listener/DbNotificationListener.java| 182 +--- .../json/JSONCreateFunctionMessage.java | 3 +- .../messaging/json/JSONDropFunctionMessage.java | 3 +- .../messaging/json/JSONMessageFactory.java | 39 +- .../listener/TestDbNotificationListener.java| 14 +- .../TestReplAcidTablesWithJsonMessage.java | 43 ++ ...eplAcrossInstancesWithJsonMessageFormat.java | 45 ++ ...ncrementalLoadAcidTablesWithJsonMessage.java | 46 ++ .../ql/parse/TestReplWithJsonMessageFormat.java | 39 ++ .../hive/ql/parse/TestReplicationScenarios.java | 82 ++-- .../TestReplicationScenariosAcidTables.java | 61 +-- ...TestReplicationScenariosAcrossInstances.java | 103 +++-- ...ationScenariosIncrementalLoadAcidTables.java | 55 ++- .../hadoop/hive/ql/parse/WarehouseInstance.java | 2 +- .../ql/cache/results/QueryResultsCache.java | 14 +- .../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 5 +- .../repl/bootstrap/load/LoadConstraint.java | 4 +- .../parse/repl/dump/events/AbortTxnHandler.java | 12 +- .../events/AbstractConstraintEventHandler.java | 3 +- .../repl/dump/events/AbstractEventHandler.java | 32 +- .../repl/dump/events/AddForeignKeyHandler.java | 12 +- .../events/AddNotNullConstraintHandler.java | 12 +- .../repl/dump/events/AddPartitionHandler.java | 10 +- .../repl/dump/events/AddPrimaryKeyHandler.java | 12 +- .../dump/events/AddUniqueConstraintHandler.java | 13 +- .../repl/dump/events/AllocWriteIdHandler.java | 12 +- .../repl/dump/events/AlterDatabaseHandler.java | 12 +- .../repl/dump/events/AlterPartitionHandler.java | 21 +- .../repl/dump/events/AlterTableHandler.java | 18 +- .../repl/dump/events/CommitTxnHandler.java | 28 +- .../repl/dump/events/CreateDatabaseHandler.java | 13 +- .../repl/dump/events/CreateFunctionHandler.java | 13 +- .../repl/dump/events/CreateTableHandler.java| 15 +- .../parse/repl/dump/events/DefaultHandler.java | 9 + .../repl/dump/events/DropConstraintHandler.java | 13 +- .../repl/dump/events/DropDatabaseHandler.java | 12 +- .../repl/dump/events/DropFunctionHandler.java | 12 +- .../repl/dump/events/DropPartitionHandler.java | 12 +- .../repl/dump/events/DropTableHandler.java | 12 +- .../repl/dump/events/EventHandlerFactory.java | 44 +- .../parse/repl/dump/events/InsertHandler.java | 22 +- .../parse/repl/dump/events/OpenTxnHandler.java | 12 +- .../repl/dump/io/ConstraintsSerializer.java | 10 +- .../load/message/AbstractMessageHandler.java| 4 +- .../dump/events/TestEventHandlerFactory.java| 7 +- .../hive/metastore/conf/MetastoreConf.java | 2 +- .../hive/metastore/messaging/EventMessage.java | 64 +-- .../metastore/messaging/MessageBuilder.java | 425 ++ .../metastore/messaging/MessageEncoder.java | 27 ++ .../metastore/messaging/MessageFactory.java | 367 +++- .../metastore/messaging/MessageSerializer.java | 24 ++ .../event/filters/DatabaseAndTableFilter.java | 8 +- .../messaging/json/JSONAcidWriteMessage.java| 9 +- .../json/JSONAddForeignKeyMessage.java | 5 +- .../json/JSONAddNotNullConstraintMessage.java | 5 +- .../messaging/json/JSONAddPartitionMessage.java | 11 +- .../json/JSONAddPrimaryKeyMessage.java | 5 +- .../json/JSONAddUniqueConstraintMessage.java| 5 +- .../messaging/json/JSONAlterCatalogMessage.java | 9 +- .../json/JSONAlterDatabaseMessage.java | 9 +- .../json/JSONAlterPartitionMessage.java | 15 +- .../messaging/json/JSONAlterTableMessage.java | 9 +- .../messaging/json/JSONCommitTxnMessage.java| 5 +- .../json/JSONCreateDatabaseMessage.java | 5 +- .../json/JSONCreateFunctionMessage.java | 5 +- .../messaging/json/JSONCreateTableMessage.java | 5 +- .../json/JSONDropPartitionMessage.java | 5 +- .../messaging/json/JSONDropTableMessage.java| 5 +- .../messaging/json/JSONInsertMessage.java | 9 +- .../messaging/json/JSONMessageEncoder.java | 70 +++ .../messaging/json/JSONMessageFactory.java | 432 --- .../messaging/json/gzip/DeSerializer.java | 181
