ayushtkn commented on a change in pull request #2539:
URL: https://github.com/apache/hive/pull/2539#discussion_r766370403
##########
File path: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
##########
@@ -695,6 +695,10 @@ private static void populateLlapDaemonVarsSet(Set<String>
llapDaemonVarsSetLocal
+ "data copy, the target data is overwritten and the modifications are
removed and the copy is again "
+ "attempted using the snapshot based approach. If disabled, the
replication will fail in case the target is "
+ "modified."),
+ REPL_REUSE_SNAPSHOTS("hive.repl.reuse.snapshots", false,
+ "If enabled,reusing snapshots is attempted in case of controlled
failover(B->A) when same paths are"
+ + "used for external table replication on src and target. Also in
cases of failed incremental where re-bootstrap is required."
+ + "If set to true and snapshots exist in some paths, it creates/reuses
new snapshots in those paths using the same name as exisiting snapshots."),
Review comment:
Typo:
exisiting
##########
File path:
shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
##########
@@ -1237,25 +1238,42 @@ public boolean runDistCpWithSnapshots(String
oldSnapshot, String newSnapshot, Li
LOG.warn("Copy failed with INVALID_ARGUMENT for source: {} to target:
{} snapshot1: {} snapshot2: {} "
+ "params: {}", srcPaths, dst, oldSnapshot, newSnapshot, params);
return true;
- } else if (returnCode == DistCpConstants.UNKNOWN_ERROR &&
overwriteTarget) {
+ } else if (returnCode == DistCpConstants.UNKNOWN_ERROR) {
// Check if this error is due to target modified.
- if (shouldRdiff(dst, conf, oldSnapshot, overwriteTarget)) {
- LOG.warn("Copy failed due to target modified. Attempting to restore
back the target. source: {} target: {} "
- + "snapshot: {}", srcPaths, dst, oldSnapshot);
- List<String> rParams = constructDistCpWithSnapshotParams(srcPaths,
dst, ".", oldSnapshot, conf, "-rdiff");
- DistCp rDistcp = new DistCp(conf, null);
- returnCode = rDistcp.run(rParams.toArray(new String[0]));
- if (returnCode == 0) {
- LOG.info("Target restored to previous state. source: {} target:
{} snapshot: {}. Reattempting to copy.",
- srcPaths, dst, oldSnapshot);
- dst.getFileSystem(conf).deleteSnapshot(dst, oldSnapshot);
- dst.getFileSystem(conf).createSnapshot(dst, oldSnapshot);
- returnCode = distcp.run(params.toArray(new String[0]));
+ if (targetModified(dst, conf, oldSnapshot)) {
+ if (overwriteTarget) {
+ LOG.warn("Copy failed due to target modified. Attempting to
restore back the target. source: {} target: {} "
+ + "snapshot: {}", srcPaths, dst, oldSnapshot);
+ List<String> rParams = constructDistCpWithSnapshotParams(srcPaths,
dst, ".", oldSnapshot, conf, "-rdiff");
+ DistCp rDistcp = new DistCp(conf, null);
+ returnCode = rDistcp.run(rParams.toArray(new String[0]));
if (returnCode == 0) {
+ LOG.info("Target restored to previous state. source: {} target:
{} snapshot: {}. Reattempting to copy.",
+ srcPaths, dst, oldSnapshot);
+ dst.getFileSystem(conf).deleteSnapshot(dst, oldSnapshot);
+ dst.getFileSystem(conf).createSnapshot(dst, oldSnapshot);
+ returnCode = distcp.run(params.toArray(new String[0]));
+ if (returnCode == 0) {
+ return true;
+ } else {
+ LOG.error("Copy failed with after target restore for source:
{} to target: {} snapshot1: {} snapshot2: "
+ + "{} params: {}. Return code: {}", srcPaths, dst,
oldSnapshot, newSnapshot, params, returnCode);
+ return false;
+ }
+ }
+ } else {
+ //in case overwriteTarget is false, and we encounter an exception
due to targetFs getting modified
+ // since last snapshot, then fall back to full distcp
+ LOG.warn("Copy failed due to target modified and overwrite is
false. Attempting full distcp." +
+ "Source:{}, target: {}",srcPaths, dst);
+ // Get the path relative to the initial snapshot for copy.
+ Path snapRelPath = new Path(srcPaths.get(0),
HdfsConstants.DOT_SNAPSHOT_DIR + "/" + newSnapshot);
+ // Copy from the initial snapshot path.
+ if(runDistCp(Collections.singletonList(snapRelPath), dst, conf)) {
return true;
} else {
- LOG.error("Copy failed with after target restore for source: {}
to target: {} snapshot1: {} snapshot2: "
- + "{} params: {}. Return code: {}", srcPaths, dst,
oldSnapshot, newSnapshot, params, returnCode);
+ LOG.error("Copy failed with full distcp for source: {} to
target: {} snapshot:"
+ + "{} params: {}. Return code: {}", srcPaths, dst,
newSnapshot, returnCode);
Review comment:
Incorrect number of placeholders. Misses params
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
