aasha commented on a change in pull request #1479:
URL: https://github.com/apache/hive/pull/1479#discussion_r485324479
##########
File path:
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
##########
@@ -2018,6 +2018,129 @@ public void
testIncrementalLoadWithPreviousDumpDeleteFailed() throws IOException
verifyRun("SELECT a from " + replDbName + ".ptned WHERE b=2", ptnData2,
driverMirror);
}
+ @Test
+ public void testConfiguredDeleteOfPrevDumpDir() throws IOException {
+ boolean verifySetupOriginal = verifySetupSteps;
+ verifySetupSteps = true;
+ String nameOfTest = "testConfigureDeleteOfPrevDumpDir";
Review comment:
String name = testName.getMethodName(); can be used
##########
File path:
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
##########
@@ -2018,6 +2018,129 @@ public void
testIncrementalLoadWithPreviousDumpDeleteFailed() throws IOException
verifyRun("SELECT a from " + replDbName + ".ptned WHERE b=2", ptnData2,
driverMirror);
}
+ @Test
+ public void testConfiguredDeleteOfPrevDumpDir() throws IOException {
+ boolean verifySetupOriginal = verifySetupSteps;
+ verifySetupSteps = true;
+ String nameOfTest = "testConfigureDeleteOfPrevDumpDir";
+ String dbName = createDB(nameOfTest, driver);
+ String replDbName = dbName + "_dupe";
+ List<String> withConfigDeletePrevDump = Arrays.asList(
+ "'" + HiveConf.ConfVars.REPL_RETAIN_PREV_DUMP_DIR + "'= 'false' ");
+ List<String> withConfigRetainPrevDump = Arrays.asList(
+ "'" + HiveConf.ConfVars.REPL_RETAIN_PREV_DUMP_DIR + "'= 'true' ",
+ "'" + HiveConf.ConfVars.REPL_RETAIN_PREV_DUMP_DIR_COUNT + "'= '2'
");
+
+ run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE",
driver);
+ run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int)
STORED AS TEXTFILE", driver);
+
+ Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName);
+
+ String[] unptnData = new String[] {"eleven", "twelve"};
+ String[] ptnData1 = new String[] {"thirteen", "fourteen", "fifteen"};
+ String[] ptnData2 = new String[] {"fifteen", "sixteen", "seventeen"};
+ String[] empty = new String[] {};
+
+ String unptnLocn = new Path(TEST_PATH, nameOfTest +
"_unptn").toUri().getPath();
+ String ptnLocn1 = new Path(TEST_PATH, nameOfTest +
"_ptn1").toUri().getPath();
+ String ptnLocn2 = new Path(TEST_PATH, nameOfTest +
"_ptn2").toUri().getPath();
+
+ createTestDataFile(unptnLocn, unptnData);
+ createTestDataFile(ptnLocn1, ptnData1);
+ createTestDataFile(ptnLocn2, ptnData2);
+
+ run("LOAD DATA LOCAL INPATH '" + unptnLocn + "' OVERWRITE INTO TABLE " +
dbName + ".unptned", driver);
+ verifySetup("SELECT * from " + dbName + ".unptned", unptnData, driver);
+ run("CREATE TABLE " + dbName + ".unptned_late LIKE " + dbName +
".unptned", driver);
+ run("INSERT INTO TABLE " + dbName + ".unptned_late SELECT * FROM " +
dbName + ".unptned", driver);
+ verifySetup("SELECT * from " + dbName + ".unptned_late", unptnData,
driver);
+
+ //perform first incremental with default option and check that
bootstrap-dump-dir gets deleted
+ Path bootstrapDumpDir = new Path(bootstrapDump.dumpLocation,
ReplUtils.REPL_HIVE_BASE_DIR);
+ FileSystem fs = FileSystem.get(bootstrapDumpDir.toUri(), hconf);
+ assertTrue(fs.exists(bootstrapDumpDir));
+ Tuple incrDump = replDumpDb(dbName);
+ assertFalse(fs.exists(bootstrapDumpDir));
+
+
+ loadAndVerify(replDbName, dbName, incrDump.lastReplId);
+ verifyRun("SELECT * from " + replDbName + ".unptned", unptnData,
driverMirror);
+ verifyRun("SELECT * from " + replDbName + ".unptned_late", unptnData,
driverMirror);
+
+ run("ALTER TABLE " + dbName + ".ptned ADD PARTITION (b=1)", driver);
+ run("LOAD DATA LOCAL INPATH '" + ptnLocn1 + "' OVERWRITE INTO TABLE " +
dbName
+ + ".ptned PARTITION(b=1)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", ptnData1,
driver);
+ run("LOAD DATA LOCAL INPATH '" + ptnLocn2 + "' OVERWRITE INTO TABLE " +
dbName
+ + ".ptned PARTITION(b=2)", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptnData2,
driver);
+
+
+ //Perform 2nd incremental with retain option.
+ //Check 1st incremental dump-dir is present even after 2nd incr dump.
+ Path incrDumpDir1 = new Path(incrDump.dumpLocation,
ReplUtils.REPL_HIVE_BASE_DIR);
+ incrDump = replDumpDb(dbName, withConfigRetainPrevDump);
+ assertTrue(fs.exists(incrDumpDir1));
+
+ loadAndVerify(replDbName, dbName, incrDump.lastReplId);
+ verifyRun("SELECT a from " + replDbName + ".ptned WHERE b=1", ptnData1,
driverMirror);
+ verifyRun("SELECT a from " + replDbName + ".ptned WHERE b=2", ptnData2,
driverMirror);
+
+ run("CREATE TABLE " + dbName
+ + ".ptned_late(a string) PARTITIONED BY (b int) STORED AS
TEXTFILE", driver);
+ run("INSERT INTO TABLE " + dbName + ".ptned_late PARTITION(b=1) SELECT a
FROM " + dbName
+ + ".ptned WHERE b=1", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned_late WHERE b=1", ptnData1,
driver);
+
+
+ //Perform 3rd incremental with retain option, retaining last 2 consumed
dump-dirs.
+ //Verify 1st and 2nd incr-dump-dirs are present after 3rd incr-dump
+ Path incrDumpDir2 = new Path(incrDump.dumpLocation,
ReplUtils.REPL_HIVE_BASE_DIR);
+ incrDump = replDumpDb(dbName, withConfigRetainPrevDump);
+ assertTrue(fs.exists(incrDumpDir1));
+ assertTrue(fs.exists(incrDumpDir2));
+
+ loadAndVerify(replDbName, dbName, incrDump.lastReplId);
+ verifyRun("SELECT a from " + replDbName + ".ptned_late WHERE b=1",
ptnData1, driverMirror);
+
+
+ run("INSERT INTO TABLE " + dbName + ".ptned_late PARTITION(b=2) SELECT a
FROM " + dbName
+ + ".ptned WHERE b=2", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned_late WHERE b=2", ptnData2,
driver);
+
+
+ //perform 4'th incr-dump with retain option in policy, retaining only last
2 dump-dirs
+ //verify incr-1 dumpdir gets deleted, incr-2 and incr-3 remain
+ Path incrDumpDir3 = new Path(incrDump.dumpLocation,
ReplUtils.REPL_HIVE_BASE_DIR);
+ incrDump = replDumpDb(dbName, withConfigRetainPrevDump);
+ assertFalse(fs.exists(incrDumpDir1));
+ assertTrue(fs.exists(incrDumpDir2));
+ assertTrue(fs.exists(incrDumpDir3));
+
+ loadAndVerify(replDbName, dbName, incrDump.lastReplId);
+ verifyRun("SELECT a from " + replDbName + ".ptned_late WHERE b=2",
ptnData2, driverMirror);
+
+ run("INSERT INTO TABLE " + dbName + ".ptned_late PARTITION(b=3) SELECT a
FROM " + dbName
+ + ".ptned WHERE b=2", driver);
+ verifySetup("SELECT a from " + dbName + ".ptned_late WHERE b=3", ptnData2,
driver);
+
+
+ //ensure 4'th incr-dump dir is present
+ //perform 5'th incr-dump with retain option to be false
+ //verify all prev dump-dirs get deleted
+ Path incrDumpDir4 = new Path(incrDump.dumpLocation,
ReplUtils.REPL_HIVE_BASE_DIR);
+ assertTrue(fs.exists(incrDumpDir4));
+ incrDump = replDumpDb(dbName, withConfigDeletePrevDump);
+ assertFalse(fs.exists(incrDumpDir2));
+ assertFalse(fs.exists(incrDumpDir3));
+ assertFalse(fs.exists(incrDumpDir4));
Review comment:
incrDumpDir4 should be present. why is it deleted. after load, do a new
dump without passing any config. it should retain only current dump.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscr...@hive.apache.org
For additional commands, e-mail: gitbox-h...@hive.apache.org
