ashutosh-bapat commented on a change in pull request #587: HIVE-21213 : Acid
table bootstrap replication needs to handle directory created by compaction
with txn id
URL: https://github.com/apache/hive/pull/587#discussion_r354286299
##########
File path:
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java
##########
@@ -670,4 +678,63 @@ public void testMultiDBTxn() throws Throwable {
replica.run("drop database " + dbName1 + " cascade");
replica.run("drop database " + dbName2 + " cascade");
}
+
+ private void runCompaction(String dbName, String tblName, CompactionType
compactionType) throws Throwable {
+ HiveConf hiveConf = new HiveConf(primary.getConf());
+ TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
+ txnHandler.compact(new CompactionRequest(dbName, tblName, compactionType));
+ hiveConf.setBoolVar(HiveConf.ConfVars.COMPACTOR_CRUD_QUERY_BASED, false);
+ runWorker(hiveConf);
+ runCleaner(hiveConf);
+ }
+
+ private FileStatus[] getDirsInTableLoc(WarehouseInstance wh, String db,
String table) throws Throwable {
+ Path tblLoc = new Path(wh.getTable(db, table).getSd().getLocation());
+ FileSystem fs = tblLoc.getFileSystem(wh.getConf());
+ return fs.listStatus(tblLoc, EximUtil.getDirectoryFilter(fs));
+ }
+
+ @Test
+ public void testAcidTablesBootstrapWithCompaction() throws Throwable {
+ String tableName = testName.getMethodName();
+ primary.run("use " + primaryDbName)
+ .run("create table " + tableName + " (id int) clustered by(id)
into 3 buckets stored as orc " +
+ "tblproperties (\"transactional\"=\"true\")")
+ .run("insert into " + tableName + " values(1)")
+ .run("insert into " + tableName + " values(2)");
+ runCompaction(primaryDbName, tableName, CompactionType.MAJOR);
+ WarehouseInstance.Tuple bootstrapDump = primary.dump(primaryDbName, null);
+ replica.load(replicatedDbName, bootstrapDump.dumpLocation);
+ replica.run("use " + replicatedDbName)
+ .run("show tables")
+ .verifyResults(new String[] {tableName})
+ .run("repl status " + replicatedDbName)
+ .verifyResult(bootstrapDump.lastReplicationId)
+ .run("select id from " + tableName + " order by id")
+ .verifyResults(new String[]{"1", "2"});
+
+ FileStatus[] dirsInLoadPath = getDirsInTableLoc(primary, primaryDbName,
tableName);
+ long writeId = -1;
+ for (FileStatus fileStatus : dirsInLoadPath) {
+ if (fileStatus.getPath().getName().startsWith(AcidUtils.BASE_PREFIX)) {
+ writeId =
AcidUtils.ParsedBase.parseBase(fileStatus.getPath()).getWriteId();
+
assertTrue(AcidUtils.getVisibilityTxnId(fileStatus.getPath().getName()) != -1);
+ break;
+ }
+ }
+ //compaction is done so there should be a base directory.
+ assertTrue(writeId != -1);
+
+ dirsInLoadPath = getDirsInTableLoc(replica, replicatedDbName, tableName);
+ for (FileStatus fileStatus : dirsInLoadPath) {
+ if (fileStatus.getPath().getName().startsWith(AcidUtils.BASE_PREFIX)) {
+ assertTrue(writeId ==
AcidUtils.ParsedBase.parseBase(fileStatus.getPath()).getWriteId());
+
assertTrue(AcidUtils.getVisibilityTxnId(fileStatus.getPath().getName()) == -1);
+ writeId = -1;
+ break;
+ }
+ }
+ //make sure that it has done the verification.
+ assertTrue(writeId == -1);
Review comment:
Using writeId again for verification saves a variable but it's not so much
readable. May be you want to save writeId on the replica in a separate variable
and compare the writeId from source that on the target.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
