[2/2] hive git commit: HIVE-20582: Make hflush in hive proto logging configurable (Prasanth Jayachandran reviewed by Thejas M Nair)
HIVE-20582: Make hflush in hive proto logging configurable (Prasanth
Jayachandran reviewed by Thejas M Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/36c33ca0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/36c33ca0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/36c33ca0
Branch: refs/heads/branch-3
Commit: 36c33ca066c99dfdb21223a711c0c3f33c85b943
Parents: 29315fc
Author: Prasanth Jayachandran
Authored: Tue Sep 18 13:10:07 2018 -0700
Committer: Prasanth Jayachandran
Committed: Tue Sep 18 13:20:02 2018 -0700
--
.../java/org/apache/hadoop/hive/conf/HiveConf.java| 3 +++
.../hadoop/hive/ql/hooks/HiveProtoLoggingHook.java| 14 +-
2 files changed, 16 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/36c33ca0/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 9523640..4ec6368 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -632,6 +632,9 @@ public class HiveConf extends Configuration {
HIVE_PROTO_EVENTS_TTL("hive.hook.proto.events.ttl", "7d",
new TimeValidator(TimeUnit.DAYS),
"Time-To-Live (TTL) of proto event files before cleanup."),
+HIVE_PROTO_FILE_PER_EVENT("hive.hook.proto.file.per.event", false,
+ "Whether each proto event has to be written to separate file. " +
+"(Use this for FS that does not hflush immediately like S3A)"),
// Hadoop Configuration Properties
// Properties with null values are ignored and exist only for the purpose
of giving us
http://git-wip-us.apache.org/repos/asf/hive/blob/36c33ca0/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
index 49cba4c..aa3a926 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
@@ -185,6 +185,7 @@ public class HiveProtoLoggingHook implements
ExecuteWithHookContext {
private int logFileCount = 0;
private ProtoMessageWriter writer;
private LocalDate writerDate;
+private boolean eventPerFile;
EventLogger(HiveConf conf, Clock clock) {
this.clock = clock;
@@ -196,6 +197,8 @@ public class HiveProtoLoggingHook implements
ExecuteWithHookContext {
LOG.error(ConfVars.HIVE_PROTO_EVENTS_BASE_PATH.varname + " is not set,
logging disabled.");
}
+ eventPerFile = conf.getBoolVar(ConfVars.HIVE_PROTO_FILE_PER_EVENT);
+ LOG.info("Event per file enabled: {}", eventPerFile);
DatePartitionedLogger tmpLogger = null;
try {
if (baseDir != null) {
@@ -303,7 +306,16 @@ public class HiveProtoLoggingHook implements
ExecuteWithHookContext {
writerDate =
logger.getDateFromDir(writer.getPath().getParent().getName());
}
writer.writeProto(event);
- writer.hflush();
+ if (eventPerFile) {
+if (writer != null) {
+ LOG.debug("Event per file enabled. Closing proto event file:
{}", writer.getPath());
+ IOUtils.closeQuietly(writer);
+}
+// rollover to next file
+writer = logger.getWriter(logFileName + "_" + ++logFileCount);
+ } else {
+writer.hflush();
+ }
return;
} catch (IOException e) {
// Something wrong with writer, lets close and reopen.
hive git commit: HIVE-20582: Make hflush in hive proto logging configurable (Prasanth Jayachandran reviewed by Thejas M Nair)
Repository: hive
Updated Branches:
refs/heads/master 7450ce762 -> 8ebde0441
HIVE-20582: Make hflush in hive proto logging configurable (Prasanth
Jayachandran reviewed by Thejas M Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8ebde044
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8ebde044
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8ebde044
Branch: refs/heads/master
Commit: 8ebde04411a4ca34994019db75b57ebee9c28f71
Parents: 7450ce7
Author: Prasanth Jayachandran
Authored: Tue Sep 18 13:10:07 2018 -0700
Committer: Prasanth Jayachandran
Committed: Tue Sep 18 13:10:14 2018 -0700
--
.../java/org/apache/hadoop/hive/conf/HiveConf.java| 3 +++
.../hadoop/hive/ql/hooks/HiveProtoLoggingHook.java| 14 +-
2 files changed, 16 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/8ebde044/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 4fb8a30..8a561e5 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -642,6 +642,9 @@ public class HiveConf extends Configuration {
HIVE_PROTO_EVENTS_TTL("hive.hook.proto.events.ttl", "7d",
new TimeValidator(TimeUnit.DAYS),
"Time-To-Live (TTL) of proto event files before cleanup."),
+HIVE_PROTO_FILE_PER_EVENT("hive.hook.proto.file.per.event", false,
+ "Whether each proto event has to be written to separate file. " +
+"(Use this for FS that does not hflush immediately like S3A)"),
// Hadoop Configuration Properties
// Properties with null values are ignored and exist only for the purpose
of giving us
http://git-wip-us.apache.org/repos/asf/hive/blob/8ebde044/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
index 673c858..0af30d4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java
@@ -186,6 +186,7 @@ public class HiveProtoLoggingHook implements
ExecuteWithHookContext {
private int logFileCount = 0;
private ProtoMessageWriter writer;
private LocalDate writerDate;
+private boolean eventPerFile;
EventLogger(HiveConf conf, Clock clock) {
this.clock = clock;
@@ -197,6 +198,8 @@ public class HiveProtoLoggingHook implements
ExecuteWithHookContext {
LOG.error(ConfVars.HIVE_PROTO_EVENTS_BASE_PATH.varname + " is not set,
logging disabled.");
}
+ eventPerFile = conf.getBoolVar(ConfVars.HIVE_PROTO_FILE_PER_EVENT);
+ LOG.info("Event per file enabled: {}", eventPerFile);
DatePartitionedLogger tmpLogger = null;
try {
if (baseDir != null) {
@@ -289,7 +292,16 @@ public class HiveProtoLoggingHook implements
ExecuteWithHookContext {
writerDate =
logger.getDateFromDir(writer.getPath().getParent().getName());
}
writer.writeProto(event);
- writer.hflush();
+ if (eventPerFile) {
+if (writer != null) {
+ LOG.debug("Event per file enabled. Closing proto event file:
{}", writer.getPath());
+ IOUtils.closeQuietly(writer);
+}
+// rollover to next file
+writer = logger.getWriter(logFileName + "_" + ++logFileCount);
+ } else {
+writer.hflush();
+ }
return;
} catch (IOException e) {
// Something wrong with writer, lets close and reopen.
