This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new bb3d2222fcf HIVE-26446: HiveProtoLoggingHook fails to populate
TablesWritten field for partitioned tables (Simhadri Govindappa, reviewed by
Denys Kuzmenko)
bb3d2222fcf is described below
commit bb3d2222fcfc55c7bc1e373d0ea2b2729259851c
Author: SimhadriGovindappa <[email protected]>
AuthorDate: Wed Aug 17 13:39:41 2022 +0530
HIVE-26446: HiveProtoLoggingHook fails to populate TablesWritten field for
partitioned tables (Simhadri Govindappa, reviewed by Denys Kuzmenko)
Closes #3499
---
.../hadoop/hive/ql/hooks/HiveProtoLoggingHook.java | 4 +-
.../hive/ql/hooks/TestHiveProtoLoggingHook.java | 75 ++++++++++++++++++++++
2 files changed, 78 insertions(+), 1 deletion(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
index 640a1ca97bc..904dd4bebde 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.hooks;
+import static org.apache.hadoop.hive.ql.hooks.Entity.Type.PARTITION;
+import static org.apache.hadoop.hive.ql.hooks.Entity.Type.TABLE;
import static org.apache.hadoop.hive.ql.plan.HiveOperation.ALTERDATABASE;
import static org.apache.hadoop.hive.ql.plan.HiveOperation.ALTERDATABASE_OWNER;
import static
org.apache.hadoop.hive.ql.plan.HiveOperation.ALTERPARTITION_BUCKETNUM;
@@ -487,7 +489,7 @@ public class HiveProtoLoggingHook implements
ExecuteWithHookContext {
private List<String> getTablesFromEntitySet(Set<? extends Entity>
entities) {
List<String> tableNames = new ArrayList<>();
for (Entity entity : entities) {
- if (entity.getType() == Entity.Type.TABLE) {
+ if (entity.getType() == TABLE || entity.getType() == PARTITION) {
tableNames.add(entity.getTable().getDbName() + "." +
entity.getTable().getTableName());
}
}
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java
b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java
index ff282dd6a81..100a672f9b6 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.hive.ql.hooks;
import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
import java.util.HashSet;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -31,6 +33,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.mr.ExecDriver;
@@ -43,6 +47,8 @@ import org.apache.hadoop.hive.ql.hooks.HookContext.HookType;
import org.apache.hadoop.hive.ql.hooks.proto.HiveHookEvents.HiveHookEventProto;
import org.apache.hadoop.hive.ql.hooks.proto.HiveHookEvents.MapFieldEntry;
import org.apache.hadoop.hive.ql.log.PerfLogger;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.TezWork;
@@ -123,6 +129,16 @@ public class TestHiveProtoLoggingHook {
assertOtherInfo(event, OtherInfoType.QUERY, null);
}
+ @Test
+ public void testNonPartionedTable() throws Exception {
+ testTablesWritten(new WriteEntity(newTable(false),
WriteEntity.WriteType.INSERT), false);
+ }
+
+ @Test
+ public void testPartitionedTable() throws Exception {
+ testTablesWritten(addPartitionOutput(newTable(true),
WriteEntity.WriteType.INSERT), true);
+ }
+
@Test
public void testQueueLogs() throws Exception {
context.setHookType(HookType.PRE_EXEC_HOOK);
@@ -322,4 +338,63 @@ public class TestHiveProtoLoggingHook {
Assert.assertEquals(value, val);
}
}
+
+ private void testTablesWritten(WriteEntity we, boolean isPartitioned) throws
Exception {
+ String query = isPartitioned ?
+ "insert into test_partition partition(dt = '20220102', lable =
'test1') values('20220103', 'banana');" :
+ "insert into default.testTable1 values('ab')";
+ HashSet<WriteEntity> tableWritten = new HashSet<>();
+ tableWritten.add(we);
+ QueryState state = new QueryState.Builder().withHiveConf(conf).build();
+ @SuppressWarnings("serial")
+ QueryPlan queryPlan = new QueryPlan(HiveOperation.QUERY) {
+ };
+ queryPlan.setQueryId("test_queryId");
+ queryPlan.setQueryStartTime(1234L);
+ queryPlan.setQueryString(query);
+ queryPlan.setRootTasks(new ArrayList<>());
+ queryPlan.setInputs(new HashSet<>());
+ queryPlan.setOutputs(tableWritten);
+ PerfLogger perf = PerfLogger.getPerfLogger(conf, true);
+ HookContext ctx = new HookContext(queryPlan, state, null, "test_user",
"192.168.10.11",
+ "hive_addr", "test_op_id", "test_session_id", "test_thread_id",
true, perf, null);
+
+ ctx.setHookType(HookType.PRE_EXEC_HOOK);
+ EventLogger evtLogger = new EventLogger(conf, SystemClock.getInstance());
+ evtLogger.handle(ctx);
+ evtLogger.shutdown();
+
+ HiveHookEventProto event = loadEvent(conf, tmpFolder);
+
+ Assert.assertEquals(EventType.QUERY_SUBMITTED.name(),
event.getEventType());
+ Assert.assertEquals(we.getTable().getFullyQualifiedName(),
event.getTablesWritten(0));
+ }
+
+ private Table newTable(boolean isPartitioned) {
+ Table t = new Table("default", "testTable");
+ if (isPartitioned) {
+ FieldSchema fs = new FieldSchema();
+ fs.setName("version");
+ fs.setType("String");
+ List<FieldSchema> partCols = new ArrayList<FieldSchema>(1);
+ partCols.add(fs);
+ t.setPartCols(partCols);
+ }
+ Map<String, String> tblProps = t.getParameters();
+ if (tblProps == null) {
+ tblProps = new HashMap<>();
+ }
+ tblProps.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true");
+ t.setParameters(tblProps);
+ return t;
+ }
+
+ private WriteEntity addPartitionOutput(Table t, WriteEntity.WriteType
writeType) throws Exception {
+ Map<String, String> partSpec = new HashMap<String, String>();
+ partSpec.put("version", Integer.toString(1));
+ Partition p = new Partition(t, partSpec, new Path("/dev/null"));
+ WriteEntity we = new WriteEntity(p, writeType);
+ return we;
+ }
+
}
