pkumarsinha commented on a change in pull request #2121:
URL: https://github.com/apache/hive/pull/2121#discussion_r671981117
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
##########
@@ -431,7 +484,9 @@ private boolean shouldDump(Path previousDumpPath) throws
IOException {
return true;
} else {
FileSystem fs = previousDumpPath.getFileSystem(conf);
- return fs.exists(new Path(previousDumpPath,
LOAD_ACKNOWLEDGEMENT.toString()));
+ boolean isDumpFailoverReady = fs.exists(new Path(previousDumpPath,
ReplAck.FAILOVER_READY_MARKER.toString()))
+ && conf.getBoolVar(HiveConf.ConfVars.HIVE_REPL_FAILOVER_START);
Review comment:
Change the order of the checks. Put
conf.getBoolVar(HiveConf.ConfVars.HIVE_REPL_FAILOVER_START) first
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
##########
@@ -173,22 +177,39 @@ public int execute() {
return
ErrorMsg.REPL_FAILED_WITH_NON_RECOVERABLE_ERROR.getErrorCode();
}
Path previousValidHiveDumpPath =
getPreviousValidDumpMetadataPath(dumpRoot);
- boolean isBootstrap = (previousValidHiveDumpPath == null);
- work.setBootstrap(isBootstrap);
+ work.setBootstrap(previousValidHiveDumpPath == null);
if (previousValidHiveDumpPath != null) {
work.setOldReplScope(new DumpMetaData(previousValidHiveDumpPath,
conf).getReplScope());
}
//If no previous dump is present or previous dump is already loaded,
proceed with the dump operation.
if (shouldDump(previousValidHiveDumpPath)) {
- Path currentDumpPath = getCurrentDumpPath(dumpRoot, isBootstrap);
+ Database db = getHive().getDatabase(work.dbNameOrPattern);
Review comment:
Move this logic to a method
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
##########
@@ -173,22 +177,39 @@ public int execute() {
return
ErrorMsg.REPL_FAILED_WITH_NON_RECOVERABLE_ERROR.getErrorCode();
}
Path previousValidHiveDumpPath =
getPreviousValidDumpMetadataPath(dumpRoot);
- boolean isBootstrap = (previousValidHiveDumpPath == null);
- work.setBootstrap(isBootstrap);
+ work.setBootstrap(previousValidHiveDumpPath == null);
if (previousValidHiveDumpPath != null) {
work.setOldReplScope(new DumpMetaData(previousValidHiveDumpPath,
conf).getReplScope());
}
//If no previous dump is present or previous dump is already loaded,
proceed with the dump operation.
if (shouldDump(previousValidHiveDumpPath)) {
- Path currentDumpPath = getCurrentDumpPath(dumpRoot, isBootstrap);
+ Database db = getHive().getDatabase(work.dbNameOrPattern);
+ boolean isDbBeingFailedOver = MetaStoreUtils.isDbBeingFailedOver(db);
+ if (!work.isBootstrap()) {
+ FileSystem fs = previousValidHiveDumpPath.getFileSystem(conf);
+ Path failoverReadyMarkerFile = new Path(previousValidHiveDumpPath,
ReplAck.FAILOVER_READY_MARKER.toString());
+ if (fs.exists(failoverReadyMarkerFile)) {
+ if (isDbBeingFailedOver) {
+ LOG.info("Deleting previous failover ready marker file: " +
failoverReadyMarkerFile);
+ fs.delete(failoverReadyMarkerFile, true);
+ } else {
+ LOG.info("Switching to bootstrap dump as this is the first
dump execution after failover.");
+ work.setFirstDumpAfterFailover(true);
+ }
+ }
+ }
+ if (isDbBeingFailedOver &&
!conf.getBoolVar(HiveConf.ConfVars.HIVE_REPL_FAILOVER_START)) {
Review comment:
Need to refactor this code.Document the cases. Refactor the method to
reuse the outcome of fs.exists() calls. Handle the failure cases.
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
##########
@@ -173,22 +177,39 @@ public int execute() {
return
ErrorMsg.REPL_FAILED_WITH_NON_RECOVERABLE_ERROR.getErrorCode();
}
Path previousValidHiveDumpPath =
getPreviousValidDumpMetadataPath(dumpRoot);
- boolean isBootstrap = (previousValidHiveDumpPath == null);
- work.setBootstrap(isBootstrap);
+ work.setBootstrap(previousValidHiveDumpPath == null);
if (previousValidHiveDumpPath != null) {
work.setOldReplScope(new DumpMetaData(previousValidHiveDumpPath,
conf).getReplScope());
}
//If no previous dump is present or previous dump is already loaded,
proceed with the dump operation.
if (shouldDump(previousValidHiveDumpPath)) {
- Path currentDumpPath = getCurrentDumpPath(dumpRoot, isBootstrap);
+ Database db = getHive().getDatabase(work.dbNameOrPattern);
+ boolean isDbBeingFailedOver = MetaStoreUtils.isDbBeingFailedOver(db);
+ if (!work.isBootstrap()) {
+ FileSystem fs = previousValidHiveDumpPath.getFileSystem(conf);
+ Path failoverReadyMarkerFile = new Path(previousValidHiveDumpPath,
ReplAck.FAILOVER_READY_MARKER.toString());
+ if (fs.exists(failoverReadyMarkerFile)) {
+ if (isDbBeingFailedOver) {
+ LOG.info("Deleting previous failover ready marker file: " +
failoverReadyMarkerFile);
+ fs.delete(failoverReadyMarkerFile, true);
+ } else {
+ LOG.info("Switching to bootstrap dump as this is the first
dump execution after failover.");
+ work.setFirstDumpAfterFailover(true);
+ }
+ }
+ }
+ if (isDbBeingFailedOver &&
!conf.getBoolVar(HiveConf.ConfVars.HIVE_REPL_FAILOVER_START)) {
Review comment:
Would it ever happen during bootstrap?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
