Repository: hive Updated Branches: refs/heads/master 7450ce762 -> 8ebde0441
HIVE-20582: Make hflush in hive proto logging configurable (Prasanth Jayachandran reviewed by Thejas M Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8ebde044 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8ebde044 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8ebde044 Branch: refs/heads/master Commit: 8ebde04411a4ca34994019db75b57ebee9c28f71 Parents: 7450ce7 Author: Prasanth Jayachandran <prasan...@apache.org> Authored: Tue Sep 18 13:10:07 2018 -0700 Committer: Prasanth Jayachandran <prasan...@apache.org> Committed: Tue Sep 18 13:10:14 2018 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 3 +++ .../hadoop/hive/ql/hooks/HiveProtoLoggingHook.java | 14 +++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/8ebde044/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 4fb8a30..8a561e5 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -642,6 +642,9 @@ public class HiveConf extends Configuration { HIVE_PROTO_EVENTS_TTL("hive.hook.proto.events.ttl", "7d", new TimeValidator(TimeUnit.DAYS), "Time-To-Live (TTL) of proto event files before cleanup."), + HIVE_PROTO_FILE_PER_EVENT("hive.hook.proto.file.per.event", false, + "Whether each proto event has to be written to separate file. " + + "(Use this for FS that does not hflush immediately like S3A)"), // Hadoop Configuration Properties // Properties with null values are ignored and exist only for the purpose of giving us http://git-wip-us.apache.org/repos/asf/hive/blob/8ebde044/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java index 673c858..0af30d4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java @@ -186,6 +186,7 @@ public class HiveProtoLoggingHook implements ExecuteWithHookContext { private int logFileCount = 0; private ProtoMessageWriter<HiveHookEventProto> writer; private LocalDate writerDate; + private boolean eventPerFile; EventLogger(HiveConf conf, Clock clock) { this.clock = clock; @@ -197,6 +198,8 @@ public class HiveProtoLoggingHook implements ExecuteWithHookContext { LOG.error(ConfVars.HIVE_PROTO_EVENTS_BASE_PATH.varname + " is not set, logging disabled."); } + eventPerFile = conf.getBoolVar(ConfVars.HIVE_PROTO_FILE_PER_EVENT); + LOG.info("Event per file enabled: {}", eventPerFile); DatePartitionedLogger<HiveHookEventProto> tmpLogger = null; try { if (baseDir != null) { @@ -289,7 +292,16 @@ public class HiveProtoLoggingHook implements ExecuteWithHookContext { writerDate = logger.getDateFromDir(writer.getPath().getParent().getName()); } writer.writeProto(event); - writer.hflush(); + if (eventPerFile) { + if (writer != null) { + LOG.debug("Event per file enabled. Closing proto event file: {}", writer.getPath()); + IOUtils.closeQuietly(writer); + } + // rollover to next file + writer = logger.getWriter(logFileName + "_" + ++logFileCount); + } else { + writer.hflush(); + } return; } catch (IOException e) { // Something wrong with writer, lets close and reopen.
