http://git-wip-us.apache.org/repos/asf/hive/blob/cbb9233a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java index ddbbd1a..38358f2 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java @@ -89,14 +89,14 @@ public class TestTxnNoBuckets extends TxnCommandsBaseForTests { /**the insert creates 2 output files (presumably because there are 2 input files) * The number in the file name is writerId. This is the number encoded in ROW__ID.bucketId - * see {@link org.apache.hadoop.hive.ql.io.BucketCodec}*/ - Assert.assertTrue(rs.get(0), rs.get(0).startsWith("{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":0}\t0\t0\t0\t")); - Assert.assertTrue(rs.get(0), rs.get(0).endsWith("nobuckets/delta_0000019_0000019_0000/bucket_00000")); - Assert.assertTrue(rs.get(1), rs.get(1).startsWith("{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":1}\t3\t3\t3\t")); - Assert.assertTrue(rs.get(1), rs.get(1).endsWith("nobuckets/delta_0000019_0000019_0000/bucket_00000")); - Assert.assertTrue(rs.get(2), rs.get(2).startsWith("{\"transactionid\":19,\"bucketid\":536936448,\"rowid\":0}\t1\t1\t1\t")); - Assert.assertTrue(rs.get(2), rs.get(2).endsWith("nobuckets/delta_0000019_0000019_0000/bucket_00001")); - Assert.assertTrue(rs.get(3), rs.get(3).startsWith("{\"transactionid\":19,\"bucketid\":536936448,\"rowid\":1}\t2\t2\t2\t")); - Assert.assertTrue(rs.get(3), rs.get(3).endsWith("nobuckets/delta_0000019_0000019_0000/bucket_00001")); + Assert.assertTrue(rs.get(0), rs.get(0).startsWith("{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":0}\t0\t0\t0\t")); + Assert.assertTrue(rs.get(0), rs.get(0).endsWith("nobuckets/delta_0000001_0000001_0000/bucket_00000")); + Assert.assertTrue(rs.get(1), rs.get(1).startsWith("{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":1}\t3\t3\t3\t")); + Assert.assertTrue(rs.get(1), rs.get(1).endsWith("nobuckets/delta_0000001_0000001_0000/bucket_00000")); + Assert.assertTrue(rs.get(2), rs.get(2).startsWith("{\"transactionid\":1,\"bucketid\":536936448,\"rowid\":0}\t1\t1\t1\t")); + Assert.assertTrue(rs.get(2), rs.get(2).endsWith("nobuckets/delta_0000001_0000001_0000/bucket_00001")); + Assert.assertTrue(rs.get(3), rs.get(3).startsWith("{\"transactionid\":1,\"bucketid\":536936448,\"rowid\":1}\t2\t2\t2\t")); + Assert.assertTrue(rs.get(3), rs.get(3).endsWith("nobuckets/delta_0000001_0000001_0000/bucket_00001")); runStatementOnDriver("update nobuckets set c3 = 17 where c3 in(0,1)"); rs = runStatementOnDriver("select ROW__ID, c1, c2, c3, INPUT__FILE__NAME from nobuckets order by INPUT__FILE__NAME, ROW__ID"); @@ -104,22 +104,22 @@ public class TestTxnNoBuckets extends TxnCommandsBaseForTests { for(String s : rs) { LOG.warn(s); } - Assert.assertTrue(rs.get(0), rs.get(0).startsWith("{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":1}\t3\t3\t3\t")); - Assert.assertTrue(rs.get(0), rs.get(0).endsWith("nobuckets/delta_0000019_0000019_0000/bucket_00000")); - Assert.assertTrue(rs.get(1), rs.get(1).startsWith("{\"transactionid\":19,\"bucketid\":536936448,\"rowid\":1}\t2\t2\t2\t")); - Assert.assertTrue(rs.get(1), rs.get(1).endsWith("nobuckets/delta_0000019_0000019_0000/bucket_00001")); + Assert.assertTrue(rs.get(0), rs.get(0).startsWith("{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":1}\t3\t3\t3\t")); + Assert.assertTrue(rs.get(0), rs.get(0).endsWith("nobuckets/delta_0000001_0000001_0000/bucket_00000")); + Assert.assertTrue(rs.get(1), rs.get(1).startsWith("{\"transactionid\":1,\"bucketid\":536936448,\"rowid\":1}\t2\t2\t2\t")); + Assert.assertTrue(rs.get(1), rs.get(1).endsWith("nobuckets/delta_0000001_0000001_0000/bucket_00001")); //so update has 1 writer which creates bucket0 where both new rows land - Assert.assertTrue(rs.get(2), rs.get(2).startsWith("{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":0}\t0\t0\t17\t")); - Assert.assertTrue(rs.get(2), rs.get(2).endsWith("nobuckets/delta_0000021_0000021_0000/bucket_00000")); - Assert.assertTrue(rs.get(3), rs.get(3).startsWith("{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t17\t")); - Assert.assertTrue(rs.get(3), rs.get(3).endsWith("nobuckets/delta_0000021_0000021_0000/bucket_00000")); + Assert.assertTrue(rs.get(2), rs.get(2).startsWith("{\"transactionid\":2,\"bucketid\":536870912,\"rowid\":0}\t0\t0\t17\t")); + Assert.assertTrue(rs.get(2), rs.get(2).endsWith("nobuckets/delta_0000002_0000002_0000/bucket_00000")); + Assert.assertTrue(rs.get(3), rs.get(3).startsWith("{\"transactionid\":2,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t17\t")); + Assert.assertTrue(rs.get(3), rs.get(3).endsWith("nobuckets/delta_0000002_0000002_0000/bucket_00000")); Set<String> expectedFiles = new HashSet<>(); //both delete events land in a single bucket0. Each has a different ROW__ID.bucketId value (even writerId in it is different) - expectedFiles.add("ts/delete_delta_0000021_0000021_0000/bucket_00000"); - expectedFiles.add("nobuckets/delta_0000019_0000019_0000/bucket_00000"); - expectedFiles.add("nobuckets/delta_0000019_0000019_0000/bucket_00001"); - expectedFiles.add("nobuckets/delta_0000021_0000021_0000/bucket_00000"); + expectedFiles.add("ts/delete_delta_0000002_0000002_0000/bucket_00000"); + expectedFiles.add("nobuckets/delta_0000001_0000001_0000/bucket_00000"); + expectedFiles.add("nobuckets/delta_0000001_0000001_0000/bucket_00001"); + expectedFiles.add("nobuckets/delta_0000002_0000002_0000/bucket_00000"); //check that we get the right files on disk assertExpectedFileSet(expectedFiles, getWarehouseDir() + "/nobuckets"); //todo: it would be nice to check the contents of the files... could use orc.FileDump - it has @@ -133,33 +133,33 @@ public class TestTxnNoBuckets extends TxnCommandsBaseForTests { LOG.warn(s); } /* -âââ base_0000021 +âââ base_0000002 â  âââ bucket_00000 â  âââ bucket_00001 -âââ delete_delta_0000021_0000021_0000 +âââ delete_delta_0000002_0000002_0000 â  âââ bucket_00000 -âââ delta_0000019_0000019_0000 +âââ delta_0000001_0000001_0000 â  âââ bucket_00000 â  âââ bucket_00001 -âââ delta_0000021_0000021_0000 +âââ delta_0000002_0000002_0000 âââ bucket_00000 */ - Assert.assertTrue(rs.get(0), rs.get(0).startsWith("{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":1}\t3\t3\t3\t")); - Assert.assertTrue(rs.get(0), rs.get(0).endsWith("nobuckets/base_0000021/bucket_00000")); - Assert.assertTrue(rs.get(1), rs.get(1).startsWith("{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":0}\t0\t0\t17\t")); - Assert.assertTrue(rs.get(1), rs.get(1).endsWith("nobuckets/base_0000021/bucket_00000")); - Assert.assertTrue(rs.get(2), rs.get(2).startsWith("{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t17\t")); - Assert.assertTrue(rs.get(2), rs.get(2).endsWith("nobuckets/base_0000021/bucket_00000")); - Assert.assertTrue(rs.get(3), rs.get(3).startsWith("{\"transactionid\":19,\"bucketid\":536936448,\"rowid\":1}\t2\t2\t2\t")); - Assert.assertTrue(rs.get(3), rs.get(3).endsWith("nobuckets/base_0000021/bucket_00001")); + Assert.assertTrue(rs.get(0), rs.get(0).startsWith("{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":1}\t3\t3\t3\t")); + Assert.assertTrue(rs.get(0), rs.get(0).endsWith("nobuckets/base_0000002/bucket_00000")); + Assert.assertTrue(rs.get(1), rs.get(1).startsWith("{\"transactionid\":2,\"bucketid\":536870912,\"rowid\":0}\t0\t0\t17\t")); + Assert.assertTrue(rs.get(1), rs.get(1).endsWith("nobuckets/base_0000002/bucket_00000")); + Assert.assertTrue(rs.get(2), rs.get(2).startsWith("{\"transactionid\":2,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t17\t")); + Assert.assertTrue(rs.get(2), rs.get(2).endsWith("nobuckets/base_0000002/bucket_00000")); + Assert.assertTrue(rs.get(3), rs.get(3).startsWith("{\"transactionid\":1,\"bucketid\":536936448,\"rowid\":1}\t2\t2\t2\t")); + Assert.assertTrue(rs.get(3), rs.get(3).endsWith("nobuckets/base_0000002/bucket_00001")); expectedFiles.clear(); - expectedFiles.add("delete_delta_0000021_0000021_0000/bucket_00000"); - expectedFiles.add("uckets/delta_0000019_0000019_0000/bucket_00000"); - expectedFiles.add("uckets/delta_0000019_0000019_0000/bucket_00001"); - expectedFiles.add("uckets/delta_0000021_0000021_0000/bucket_00000"); - expectedFiles.add("/warehouse/nobuckets/base_0000021/bucket_00000"); - expectedFiles.add("/warehouse/nobuckets/base_0000021/bucket_00001"); + expectedFiles.add("delete_delta_0000002_0000002_0000/bucket_00000"); + expectedFiles.add("uckets/delta_0000001_0000001_0000/bucket_00000"); + expectedFiles.add("uckets/delta_0000001_0000001_0000/bucket_00001"); + expectedFiles.add("uckets/delta_0000002_0000002_0000/bucket_00000"); + expectedFiles.add("/warehouse/nobuckets/base_0000002/bucket_00000"); + expectedFiles.add("/warehouse/nobuckets/base_0000002/bucket_00001"); assertExpectedFileSet(expectedFiles, getWarehouseDir() + "/nobuckets"); TestTxnCommands2.runCleaner(hiveConf); @@ -168,8 +168,8 @@ public class TestTxnNoBuckets extends TxnCommandsBaseForTests { Assert.assertEquals("Unexpected result after clean", stringifyValues(result), rs); expectedFiles.clear(); - expectedFiles.add("nobuckets/base_0000021/bucket_00000"); - expectedFiles.add("nobuckets/base_0000021/bucket_00001"); + expectedFiles.add("nobuckets/base_0000002/bucket_00000"); + expectedFiles.add("nobuckets/base_0000002/bucket_00001"); assertExpectedFileSet(expectedFiles, getWarehouseDir() + "/nobuckets"); } @@ -185,8 +185,8 @@ public class TestTxnNoBuckets extends TxnCommandsBaseForTests { "'='true', 'transactional_properties'='default') as select a, b from " + Table.NONACIDORCTBL); List<String> rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from myctas order by ROW__ID"); String expected[][] = { - {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":0}\t3\t4", "warehouse/myctas/delta_0000015_0000015_0000/bucket_00000"}, - {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":1}\t1\t2", "warehouse/myctas/delta_0000015_0000015_0000/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":0}\t3\t4", "warehouse/myctas/delta_0000001_0000001_0000/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t2", "warehouse/myctas/delta_0000001_0000001_0000/bucket_00000"}, }; checkExpected(rs, expected, "Unexpected row count after ctas from non acid table"); @@ -195,8 +195,8 @@ public class TestTxnNoBuckets extends TxnCommandsBaseForTests { "'='true', 'transactional_properties'='default') as select a, b from " + Table.ACIDTBL);//todo: try this with acid default - it seem makeing table acid in listener is too late rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from myctas2 order by ROW__ID"); String expected2[][] = { - {"{\"transactionid\":18,\"bucketid\":536870912,\"rowid\":0}\t3\t4", "warehouse/myctas2/delta_0000018_0000018_0000/bucket_00000"}, - {"{\"transactionid\":18,\"bucketid\":536870912,\"rowid\":1}\t1\t2", "warehouse/myctas2/delta_0000018_0000018_0000/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":0}\t3\t4", "warehouse/myctas2/delta_0000001_0000001_0000/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t2", "warehouse/myctas2/delta_0000001_0000001_0000/bucket_00000"} }; checkExpected(rs, expected2, "Unexpected row count after ctas from acid table"); @@ -205,10 +205,10 @@ public class TestTxnNoBuckets extends TxnCommandsBaseForTests { " union all select a, b from " + Table.ACIDTBL); rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from myctas3 order by ROW__ID"); String expected3[][] = { - {"{\"transactionid\":20,\"bucketid\":536870912,\"rowid\":0}\t3\t4", "warehouse/myctas3/delta_0000020_0000020_0000/bucket_00000"}, - {"{\"transactionid\":20,\"bucketid\":536870912,\"rowid\":1}\t1\t2", "warehouse/myctas3/delta_0000020_0000020_0000/bucket_00000"}, - {"{\"transactionid\":20,\"bucketid\":536936448,\"rowid\":0}\t3\t4", "warehouse/myctas3/delta_0000020_0000020_0000/bucket_00001"}, - {"{\"transactionid\":20,\"bucketid\":536936448,\"rowid\":1}\t1\t2", "warehouse/myctas3/delta_0000020_0000020_0000/bucket_00001"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":0}\t3\t4", "warehouse/myctas3/delta_0000001_0000001_0000/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t2", "warehouse/myctas3/delta_0000001_0000001_0000/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536936448,\"rowid\":0}\t3\t4", "warehouse/myctas3/delta_0000001_0000001_0000/bucket_00001"}, + {"{\"transactionid\":1,\"bucketid\":536936448,\"rowid\":1}\t1\t2", "warehouse/myctas3/delta_0000001_0000001_0000/bucket_00001"}, }; checkExpected(rs, expected3, "Unexpected row count after ctas from union all query"); @@ -217,8 +217,8 @@ public class TestTxnNoBuckets extends TxnCommandsBaseForTests { " union distinct select a, b from " + Table.ACIDTBL); rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from myctas4 order by ROW__ID"); String expected4[][] = { - {"{\"transactionid\":22,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "/delta_0000022_0000022_0000/bucket_00000"}, - {"{\"transactionid\":22,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "/delta_0000022_0000022_0000/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "/delta_0000001_0000001_0000/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "/delta_0000001_0000001_0000/bucket_00000"}, }; checkExpected(rs, expected4, "Unexpected row count after ctas from union distinct query"); } @@ -268,11 +268,11 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver List<String> rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from T order by ROW__ID"); String expected[][] = { - {"{\"transactionid\":16,\"bucketid\":536870913,\"rowid\":0}\t1\t2", "/delta_0000016_0000016_0001/bucket_00000"}, - {"{\"transactionid\":16,\"bucketid\":536870913,\"rowid\":1}\t3\t4", "/delta_0000016_0000016_0001/bucket_00000"}, - {"{\"transactionid\":16,\"bucketid\":536870914,\"rowid\":0}\t7\t8", "/delta_0000016_0000016_0002/bucket_00000"}, - {"{\"transactionid\":16,\"bucketid\":536870914,\"rowid\":1}\t5\t6", "/delta_0000016_0000016_0002/bucket_00000"}, - {"{\"transactionid\":16,\"bucketid\":536870915,\"rowid\":0}\t9\t10", "/delta_0000016_0000016_0003/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870913,\"rowid\":0}\t1\t2", "/delta_0000001_0000001_0001/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870913,\"rowid\":1}\t3\t4", "/delta_0000001_0000001_0001/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870914,\"rowid\":0}\t7\t8", "/delta_0000001_0000001_0002/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870914,\"rowid\":1}\t5\t6", "/delta_0000001_0000001_0002/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870915,\"rowid\":0}\t9\t10", "/delta_0000001_0000001_0003/bucket_00000"}, }; checkExpected(rs, expected, "Unexpected row count after ctas"); } @@ -304,12 +304,12 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver //previous insert+union creates 3 data files (0-3) //insert (12,12) creates 000000_0_copy_1 String expected[][] = { - {"1\t2", "warehouse/t/000002_0"}, - {"2\t4", "warehouse/t/000002_0"}, - {"5\t6", "warehouse/t/000001_0"}, - {"6\t8", "warehouse/t/000000_0"}, - {"9\t10", "warehouse/t/000001_0"}, - {"12\t12", "warehouse/t/000000_0_copy_1"} + {"1\t2", "warehouse/t/000002_0"}, + {"2\t4", "warehouse/t/000002_0"}, + {"5\t6", "warehouse/t/000000_0"}, + {"6\t8", "warehouse/t/000001_0"}, + {"9\t10", "warehouse/t/000000_0"}, + {"12\t12", "warehouse/t/000000_0_copy_1"} }; checkExpected(rs, expected,"before converting to acid"); @@ -323,16 +323,16 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver " where a between 5 and 7"); //now we have a table with data files at multiple different levels. String expected1[][] = { - {"1\t2", "warehouse/t/000002_0"}, - {"2\t4", "warehouse/t/000002_0"}, - {"5\t6", "warehouse/t/000001_0"}, - {"6\t8", "warehouse/t/000000_0"}, - {"9\t10", "warehouse/t/000001_0"}, - {"10\t20", "warehouse/t/HIVE_UNION_SUBDIR_15/000000_0"}, - {"12\t12", "warehouse/t/000000_0_copy_1"}, - {"20\t40", "warehouse/t/HIVE_UNION_SUBDIR_15/000000_0"}, - {"50\t60", "warehouse/t/HIVE_UNION_SUBDIR_16/000000_0"}, - {"60\t80", "warehouse/t/HIVE_UNION_SUBDIR_16/000001_0"} + {"1\t2", "warehouse/t/000002_0"}, + {"2\t4", "warehouse/t/000002_0"}, + {"5\t6", "warehouse/t/000000_0"}, + {"6\t8", "warehouse/t/000001_0"}, + {"9\t10", "warehouse/t/000000_0"}, + {"10\t20", "warehouse/t/HIVE_UNION_SUBDIR_15/000000_0"}, + {"12\t12", "warehouse/t/000000_0_copy_1"}, + {"20\t40", "warehouse/t/HIVE_UNION_SUBDIR_15/000000_0"}, + {"50\t60", "warehouse/t/HIVE_UNION_SUBDIR_16/000000_0"}, + {"60\t80", "warehouse/t/HIVE_UNION_SUBDIR_16/000001_0"} }; rs = runStatementOnDriver("select a, b, INPUT__FILE__NAME from T order by a, b, INPUT__FILE__NAME"); checkExpected(rs, expected1,"before converting to acid (with multi level data layout)"); @@ -347,16 +347,16 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver logical bucket (tranche) */ String expected2[][] = { - {"{\"transactionid\":0,\"bucketid\":537001984,\"rowid\":0}\t1\t2", "warehouse/t/000002_0"}, - {"{\"transactionid\":0,\"bucketid\":537001984,\"rowid\":1}\t2\t4", "warehouse/t/000002_0"}, - {"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":1}\t5\t6", "warehouse/t/000001_0"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t6\t8", "warehouse/t/000000_0"}, - {"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":0}\t9\t10", "warehouse/t/000001_0"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t10\t20", "warehouse/t/HIVE_UNION_SUBDIR_15/000000_0"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t12\t12", "warehouse/t/000000_0_copy_1"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t20\t40", "warehouse/t/HIVE_UNION_SUBDIR_15/000000_0"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t50\t60", "warehouse/t/HIVE_UNION_SUBDIR_16/000000_0"}, - {"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":2}\t60\t80", "warehouse/t/HIVE_UNION_SUBDIR_16/000001_0"}, + {"{\"transactionid\":0,\"bucketid\":537001984,\"rowid\":0}\t1\t2", "warehouse/t/000002_0"}, + {"{\"transactionid\":0,\"bucketid\":537001984,\"rowid\":1}\t2\t4", "warehouse/t/000002_0"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t5\t6", "warehouse/t/000000_0"}, + {"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":0}\t6\t8", "warehouse/t/000001_0"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t9\t10", "warehouse/t/000000_0"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t10\t20", "warehouse/t/HIVE_UNION_SUBDIR_15/000000_0"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t12\t12", "warehouse/t/000000_0_copy_1"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t20\t40", "warehouse/t/HIVE_UNION_SUBDIR_15/000000_0"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":5}\t50\t60", "warehouse/t/HIVE_UNION_SUBDIR_16/000000_0"}, + {"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":1}\t60\t80", "warehouse/t/HIVE_UNION_SUBDIR_16/000001_0"}, }; checkExpected(rs, expected2,"after converting to acid (no compaction)"); Assert.assertEquals(0, BucketCodec.determineVersion(536870912).decodeWriterId(536870912)); @@ -368,15 +368,15 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver assertVectorized(shouldVectorize(), "delete from T where b = 8"); runStatementOnDriver("delete from T where b = 8"); String expected3[][] = { - {"{\"transactionid\":0,\"bucketid\":537001984,\"rowid\":0}\t1\t2", "warehouse/t/000002_0"}, - {"{\"transactionid\":0,\"bucketid\":537001984,\"rowid\":1}\t2\t4", "warehouse/t/000002_0"}, - {"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":1}\t5\t6", "warehouse/t/000001_0"}, - {"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":0}\t9\t10", "warehouse/t/000001_0"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t10\t20", "warehouse/t/HIVE_UNION_SUBDIR_15/000000_0"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t12\t12", "warehouse/t/000000_0_copy_1"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t20\t40", "warehouse/t/HIVE_UNION_SUBDIR_15/000000_0"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t50\t60", "warehouse/t/HIVE_UNION_SUBDIR_16/000000_0"}, - {"{\"transactionid\":24,\"bucketid\":536870912,\"rowid\":0}\t60\t88", "warehouse/t/delta_0000024_0000024_0000/bucket_00000"}, + {"{\"transactionid\":0,\"bucketid\":537001984,\"rowid\":0}\t1\t2", "warehouse/t/000002_0"}, + {"{\"transactionid\":0,\"bucketid\":537001984,\"rowid\":1}\t2\t4", "warehouse/t/000002_0"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t5\t6", "warehouse/t/000000_0"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t9\t10", "warehouse/t/000000_0"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t10\t20", "warehouse/t/HIVE_UNION_SUBDIR_15/000000_0"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t12\t12", "warehouse/t/000000_0_copy_1"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t20\t40", "warehouse/t/HIVE_UNION_SUBDIR_15/000000_0"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":5}\t50\t60", "warehouse/t/HIVE_UNION_SUBDIR_16/000000_0"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":0}\t60\t88", "warehouse/t/delta_0000001_0000001_0000/bucket_00000"}, }; rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from T order by a, b, INPUT__FILE__NAME"); checkExpected(rs, expected3,"after converting to acid (no compaction with updates)"); @@ -388,15 +388,15 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver /*Compaction preserves location of rows wrt buckets/tranches (for now)*/ String expected4[][] = { - {"{\"transactionid\":0,\"bucketid\":537001984,\"rowid\":0}\t1\t2", "warehouse/t/base_0000026/bucket_00002"}, - {"{\"transactionid\":0,\"bucketid\":537001984,\"rowid\":1}\t2\t4", "warehouse/t/base_0000026/bucket_00002"}, - {"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":1}\t5\t6", "warehouse/t/base_0000026/bucket_00001"}, - {"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":0}\t9\t10", "warehouse/t/base_0000026/bucket_00001"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t10\t20", "warehouse/t/base_0000026/bucket_00000"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t12\t12", "warehouse/t/base_0000026/bucket_00000"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t20\t40", "warehouse/t/base_0000026/bucket_00000"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t50\t60", "warehouse/t/base_0000026/bucket_00000"}, - {"{\"transactionid\":24,\"bucketid\":536870912,\"rowid\":0}\t60\t88", "warehouse/t/base_0000026/bucket_00000"}, + {"{\"transactionid\":0,\"bucketid\":537001984,\"rowid\":0}\t1\t2", "warehouse/t/base_0000002/bucket_00002"}, + {"{\"transactionid\":0,\"bucketid\":537001984,\"rowid\":1}\t2\t4", "warehouse/t/base_0000002/bucket_00002"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t5\t6", "warehouse/t/base_0000002/bucket_00000"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t9\t10", "warehouse/t/base_0000002/bucket_00000"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t10\t20", "warehouse/t/base_0000002/bucket_00000"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t12\t12", "warehouse/t/base_0000002/bucket_00000"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t20\t40", "warehouse/t/base_0000002/bucket_00000"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":5}\t50\t60", "warehouse/t/base_0000002/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":0}\t60\t88", "warehouse/t/base_0000002/bucket_00000"}, }; checkExpected(rs, expected4,"after major compact"); } @@ -414,14 +414,14 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver } /* The number of writers seems to be based on number of MR jobs for the src query. todo check number of FileSinks - warehouse/t/.hive-staging_hive_2017-09-13_08-59-28_141_6304543600372946004-1/-ext-10000/000000_0/delta_0000016_0000016_0000/bucket_00000 [length: 648] - {"operation":0,"originalTransaction":16,"bucket":536870912,"rowId":0,"currentTransaction":16,"row":{"_col0":1,"_col1":2}} - {"operation":0,"originalTransaction":16,"bucket":536870912,"rowId":1,"currentTransaction":16,"row":{"_col0":2,"_col1":4}} + warehouse/t/.hive-staging_hive_2017-09-13_08-59-28_141_6304543600372946004-1/-ext-10000/000000_0/delta_0000001_0000001_0000/bucket_00000 [length: 648] + {"operation":0,"originalTransaction":1,"bucket":536870912,"rowId":0,"currentTransaction":1,"row":{"_col0":1,"_col1":2}} + {"operation":0,"originalTransaction":1,"bucket":536870912,"rowId":1,"currentTransaction":1,"row":{"_col0":2,"_col1":4}} ________________________________________________________________________________________________________________________ - warehouse/t/.hive-staging_hive_2017-09-13_08-59-28_141_6304543600372946004-1/-ext-10000/000001_0/delta_0000016_0000016_0000/bucket_00001 [length: 658] - {"operation":0,"originalTransaction":16,"bucket":536936448,"rowId":0,"currentTransaction":16,"row":{"_col0":5,"_col1":6}} - {"operation":0,"originalTransaction":16,"bucket":536936448,"rowId":1,"currentTransaction":16,"row":{"_col0":6,"_col1":8}} - {"operation":0,"originalTransaction":16,"bucket":536936448,"rowId":2,"currentTransaction":16,"row":{"_col0":9,"_col1":10}} + warehouse/t/.hive-staging_hive_2017-09-13_08-59-28_141_6304543600372946004-1/-ext-10000/000001_0/delta_0000001_0000001_0000/bucket_00001 [length: 658] + {"operation":0,"originalTransaction":1,"bucket":536936448,"rowId":0,"currentTransaction":1,"row":{"_col0":5,"_col1":6}} + {"operation":0,"originalTransaction":1,"bucket":536936448,"rowId":1,"currentTransaction":1,"row":{"_col0":6,"_col1":8}} + {"operation":0,"originalTransaction":1,"bucket":536936448,"rowId":2,"currentTransaction":1,"row":{"_col0":9,"_col1":10}} */ rs = runStatementOnDriver("select a, b from T order by a, b"); Assert.assertEquals(stringifyValues(values), rs); @@ -468,15 +468,15 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver * Also check the file name (only) after compaction for completeness */ String[][] expected = { - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t0\t13", "bucket_00000", "000000_0_copy_1"}, - {"{\"transactionid\":18,\"bucketid\":536870912,\"rowid\":0}\t0\t15", "bucket_00000", "bucket_00000"}, - {"{\"transactionid\":20,\"bucketid\":536870912,\"rowid\":0}\t0\t17", "bucket_00000", "bucket_00000"}, - {"{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":0}\t0\t120", "bucket_00000", "bucket_00000"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "bucket_00000", "000000_0"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t1\t4", "bucket_00000", "000000_0_copy_1"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":5}\t1\t5", "bucket_00000", "000000_0_copy_1"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":6}\t1\t6", "bucket_00000", "000000_0_copy_2"}, - {"{\"transactionid\":18,\"bucketid\":536870912,\"rowid\":1}\t1\t16", "bucket_00000", "bucket_00000"} + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t0\t13", "bucket_00000", "000000_0_copy_1"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":0}\t0\t15", "bucket_00000", "bucket_00000"}, + {"{\"transactionid\":3,\"bucketid\":536870912,\"rowid\":0}\t0\t17", "bucket_00000", "bucket_00000"}, + {"{\"transactionid\":2,\"bucketid\":536870912,\"rowid\":0}\t0\t120", "bucket_00000", "bucket_00000"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "bucket_00000", "000000_0"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t1\t4", "bucket_00000", "000000_0_copy_1"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":5}\t1\t5", "bucket_00000", "000000_0_copy_1"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":6}\t1\t6", "bucket_00000", "000000_0_copy_2"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t16", "bucket_00000", "bucket_00000"} }; Assert.assertEquals("Unexpected row count before compaction", expected.length, rs.size()); for(int i = 0; i < expected.length; i++) { @@ -491,17 +491,17 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver âââ 000000_0 âââ 000000_0_copy_1 âââ 000000_0_copy_2 - âââ base_0000021 + âââ base_0000004 â  âââ bucket_00000 - âââ delete_delta_0000019_0000019_0000 + âââ delete_delta_0000002_0000002_0000 â  âââ bucket_00000 - âââ delete_delta_0000021_0000021_0000 + âââ delete_delta_0000004_0000004_0000 â  âââ bucket_00000 - âââ delta_0000018_0000018_0000 + âââ delta_0000001_0000001_0000 â  âââ bucket_00000 - âââ delta_0000019_0000019_0000 + âââ delta_0000002_0000002_0000 â  âââ bucket_00000 - âââ delta_0000020_0000020_0000 + âââ delta_0000003_0000003_0000 âââ bucket_00000 6 directories, 9 files @@ -569,8 +569,8 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver query = "select ROW__ID, a from T where b > 6 order by a"; rs = runStatementOnDriver(query); String[][] expected1 = { - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}", "6"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}", "9"} + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}", "6"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}", "9"} }; checkExpected(rs, expected1, "After conversion with VC1"); assertVectorized(shouldVectorize(), query); @@ -579,11 +579,11 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver query = "select ROW__ID, a from T where b > 0 order by a"; rs = runStatementOnDriver(query); String[][] expected2 = { - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}", "1"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}", "2"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}", "5"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}", "6"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}", "9"} + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}", "1"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}", "2"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}", "5"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}", "6"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}", "9"} }; checkExpected(rs, expected2, "After conversion with VC2"); assertVectorized(shouldVectorize(), query); @@ -593,8 +593,8 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver rs = runStatementOnDriver(query); Assert.assertEquals("", 2, rs.size()); String[][] expected3 = { - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t6", "warehouse/t/000000_0"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t9", "warehouse/t/000000_0"} + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t6", "warehouse/t/000000_0"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t9", "warehouse/t/000000_0"} }; checkExpected(rs, expected3, "After non-vectorized read"); Assert.assertEquals(0, BucketCodec.determineVersion(536870912).decodeWriterId(536870912)); @@ -606,11 +606,11 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver query = "select ROW__ID, b from T where b > 0 order by a"; rs = runStatementOnDriver(query); String[][] expected4 = { - {"{\"transactionid\":25,\"bucketid\":536870912,\"rowid\":0}","17"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}","4"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}","6"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}","8"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}","10"} + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":0}","17"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}","4"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}","6"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}","8"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}","10"} }; checkExpected(rs, expected4, "After conversion with VC4"); assertVectorized(shouldVectorize(), query); @@ -627,11 +627,11 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver query = "select ROW__ID, a, b, INPUT__FILE__NAME from T where b > 0 order by a, b"; rs = runStatementOnDriver(query); String[][] expected5 = {//the row__ids are the same after compaction - {"{\"transactionid\":25,\"bucketid\":536870912,\"rowid\":0}\t1\t17", "warehouse/t/base_0000025/bucket_00000"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t2\t4", "warehouse/t/base_0000025/bucket_00000"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t5\t6", "warehouse/t/base_0000025/bucket_00000"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t6\t8", "warehouse/t/base_0000025/bucket_00000"}, - {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t9\t10", "warehouse/t/base_0000025/bucket_00000"} + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t17", "warehouse/t/base_0000001/bucket_00000"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t2\t4", "warehouse/t/base_0000001/bucket_00000"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t5\t6", "warehouse/t/base_0000001/bucket_00000"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t6\t8", "warehouse/t/base_0000001/bucket_00000"}, + {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t9\t10", "warehouse/t/base_0000001/bucket_00000"} }; checkExpected(rs, expected5, "After major compaction"); //vectorized because there is INPUT__FILE__NAME @@ -671,14 +671,14 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver String query = "select ROW__ID, p, q, a, b, INPUT__FILE__NAME from T order by p, q, a, b"; List<String> rs = runStatementOnDriver(query); String[][] expected = { - {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":0}\t1\t1\t4\t1", "t/p=1/q=1/delta_0000015_0000015_0000/bucket_00000"}, - {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t4\t3", "t/p=1/q=1/delta_0000015_0000015_0000/bucket_00000"}, - {"{\"transactionid\":17,\"bucketid\":536870912,\"rowid\":0}\t1\t1\t5\t1", "t/p=1/q=1/delta_0000017_0000017_0000/bucket_00000"}, - {"{\"transactionid\":17,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t5\t3", "t/p=1/q=1/delta_0000017_0000017_0000/bucket_00000"}, - {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t4\t2", "t/p=1/q=2/delta_0000015_0000015_0000/bucket_00000"}, - {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t4\t4", "t/p=1/q=2/delta_0000015_0000015_0000/bucket_00000"}, - {"{\"transactionid\":17,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t5\t2", "t/p=1/q=2/delta_0000017_0000017_0000/bucket_00000"}, - {"{\"transactionid\":17,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t5\t4", "t/p=1/q=2/delta_0000017_0000017_0000/bucket_00000"} + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t1\t4\t1", "t/p=1/q=1/delta_0000001_0000001_0000/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t4\t3", "t/p=1/q=1/delta_0000001_0000001_0000/bucket_00000"}, + {"{\"transactionid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t1\t5\t1", "t/p=1/q=1/delta_0000002_0000002_0000/bucket_00000"}, + {"{\"transactionid\":2,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t5\t3", "t/p=1/q=1/delta_0000002_0000002_0000/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t4\t2", "t/p=1/q=2/delta_0000001_0000001_0000/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t4\t4", "t/p=1/q=2/delta_0000001_0000001_0000/bucket_00000"}, + {"{\"transactionid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t5\t2", "t/p=1/q=2/delta_0000002_0000002_0000/bucket_00000"}, + {"{\"transactionid\":2,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t5\t4", "t/p=1/q=2/delta_0000002_0000002_0000/bucket_00000"} }; checkExpected(rs, expected, "insert data"); @@ -689,14 +689,14 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver query = "select ROW__ID, p, q, a, b, INPUT__FILE__NAME from T order by p, q, a, b"; rs = runStatementOnDriver(query); String[][] expected2 = { - {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":0}\t1\t1\t4\t1", "t/p=1/q=1/delta_0000015_0000015_0000/bucket_00000"}, - {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t4\t3", "t/p=1/q=1/delta_0000015_0000015_0000/bucket_00000"}, - {"{\"transactionid\":17,\"bucketid\":536870912,\"rowid\":0}\t1\t1\t5\t1", "t/p=1/q=1/delta_0000017_0000017_0000/bucket_00000"}, - {"{\"transactionid\":17,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t5\t3", "t/p=1/q=1/delta_0000017_0000017_0000/bucket_00000"}, - {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t4\t2", "t/p=1/q=2/base_0000017/bucket_00000"}, - {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t4\t4", "t/p=1/q=2/base_0000017/bucket_00000"}, - {"{\"transactionid\":17,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t5\t2", "t/p=1/q=2/base_0000017/bucket_00000"}, - {"{\"transactionid\":17,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t5\t4", "t/p=1/q=2/base_0000017/bucket_00000"} + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t1\t4\t1", "t/p=1/q=1/delta_0000001_0000001_0000/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t4\t3", "t/p=1/q=1/delta_0000001_0000001_0000/bucket_00000"}, + {"{\"transactionid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t1\t5\t1", "t/p=1/q=1/delta_0000002_0000002_0000/bucket_00000"}, + {"{\"transactionid\":2,\"bucketid\":536870912,\"rowid\":1}\t1\t1\t5\t3", "t/p=1/q=1/delta_0000002_0000002_0000/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t4\t2", "t/p=1/q=2/base_0000002/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t4\t4", "t/p=1/q=2/base_0000002/bucket_00000"}, + {"{\"transactionid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t2\t5\t2", "t/p=1/q=2/base_0000002/bucket_00000"}, + {"{\"transactionid\":2,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t5\t4", "t/p=1/q=2/base_0000002/bucket_00000"} }; checkExpected(rs, expected2, "after major compaction"); @@ -720,9 +720,9 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver String query = "select ROW__ID, a, b, INPUT__FILE__NAME from T order by a, b"; List<String> rs = runStatementOnDriver(query); String[][] expected = { - //this proves data is written in Acid layout so T was made Acid - {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "t/delta_0000015_0000015_0000/bucket_00000"}, - {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "t/delta_0000015_0000015_0000/bucket_00000"} + //this proves data is written in Acid layout so T was made Acid + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "t/delta_0000001_0000001_0000/bucket_00000"}, + {"{\"transactionid\":1,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "t/delta_0000001_0000001_0000/bucket_00000"} }; checkExpected(rs, expected, "insert data"); }
http://git-wip-us.apache.org/repos/asf/hive/blob/cbb9233a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java index 2a520f4..d673be4 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java @@ -27,7 +27,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.StatsSetupConst; -import org.apache.hadoop.hive.common.ValidTxnList; +import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.CompilationOpContext; @@ -292,8 +292,12 @@ public class TestFileSinkOperator { } desc.setWriteType(writeType); desc.setGatherStats(true); - if (txnId > 0) desc.setTransactionId(txnId); - if (writeType != AcidUtils.Operation.NOT_ACID) desc.setTransactionId(1L); + if (txnId > 0) { + desc.setTableWriteId(txnId); + } + if (writeType != AcidUtils.Operation.NOT_ACID) { + desc.setTableWriteId(1L); + } FileSinkOperator op = (FileSinkOperator)OperatorFactory.get( new CompilationOpContext(), FileSinkDesc.class); @@ -699,7 +703,7 @@ public class TestFileSinkOperator { public RawReader<Row> getRawReader(Configuration conf, boolean collapseEvents, int bucket, - ValidTxnList validTxnList, + ValidWriteIdList validWriteIdList, Path baseDirectory, Path[] deltaDirectory) throws IOException { @@ -725,18 +729,18 @@ public class TestFileSinkOperator { return new RecordUpdater() { @Override - public void insert(long currentTransaction, Object row) throws IOException { + public void insert(long currentWriteId, Object row) throws IOException { addRow(row); numRecordsAdded++; } @Override - public void update(long currentTransaction, Object row) throws IOException { + public void update(long currentWriteId, Object row) throws IOException { addRow(row); } @Override - public void delete(long currentTransaction, Object row) throws IOException { + public void delete(long currentWriteId, Object row) throws IOException { addRow(row); numRecordsAdded--; } http://git-wip-us.apache.org/repos/asf/hive/blob/cbb9233a/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidInputFormat.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidInputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidInputFormat.java index 23dadd0..a1eb39d 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidInputFormat.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidInputFormat.java @@ -49,8 +49,8 @@ public class TestAcidInputFormat { deltaMetaData.readFields(mockDataInput); verify(mockDataInput, times(1)).readInt(); - assertThat(deltaMetaData.getMinTxnId(), is(1L)); - assertThat(deltaMetaData.getMaxTxnId(), is(2L)); + assertThat(deltaMetaData.getMinWriteId(), is(1L)); + assertThat(deltaMetaData.getMaxWriteId(), is(2L)); assertThat(deltaMetaData.getStmtIds().isEmpty(), is(true)); } @@ -63,8 +63,8 @@ public class TestAcidInputFormat { deltaMetaData.readFields(mockDataInput); verify(mockDataInput, times(3)).readInt(); - assertThat(deltaMetaData.getMinTxnId(), is(1L)); - assertThat(deltaMetaData.getMaxTxnId(), is(2L)); + assertThat(deltaMetaData.getMinWriteId(), is(1L)); + assertThat(deltaMetaData.getMaxWriteId(), is(2L)); assertThat(deltaMetaData.getStmtIds().size(), is(2)); assertThat(deltaMetaData.getStmtIds().get(0), is(100)); assertThat(deltaMetaData.getStmtIds().get(1), is(101)); @@ -74,8 +74,8 @@ public class TestAcidInputFormat { public void testDeltaMetaConstructWithState() throws Exception { DeltaMetaData deltaMetaData = new AcidInputFormat.DeltaMetaData(2000L, 2001L, Arrays.asList(97, 98, 99)); - assertThat(deltaMetaData.getMinTxnId(), is(2000L)); - assertThat(deltaMetaData.getMaxTxnId(), is(2001L)); + assertThat(deltaMetaData.getMinWriteId(), is(2000L)); + assertThat(deltaMetaData.getMaxWriteId(), is(2001L)); assertThat(deltaMetaData.getStmtIds().size(), is(3)); assertThat(deltaMetaData.getStmtIds().get(0), is(97)); assertThat(deltaMetaData.getStmtIds().get(1), is(98)); @@ -95,8 +95,8 @@ public class TestAcidInputFormat { deltaMetaData.readFields(mockDataInput); verify(mockDataInput, times(3)).readInt(); - assertThat(deltaMetaData.getMinTxnId(), is(1L)); - assertThat(deltaMetaData.getMaxTxnId(), is(2L)); + assertThat(deltaMetaData.getMinWriteId(), is(1L)); + assertThat(deltaMetaData.getMaxWriteId(), is(2L)); assertThat(deltaMetaData.getStmtIds().size(), is(2)); assertThat(deltaMetaData.getStmtIds().get(0), is(100)); assertThat(deltaMetaData.getStmtIds().get(1), is(101)); http://git-wip-us.apache.org/repos/asf/hive/blob/cbb9233a/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java index 5632350..c51ffa8 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java @@ -27,13 +27,12 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.ValidCompactorTxnList; -import org.apache.hadoop.hive.common.ValidReadTxnList; +import org.apache.hadoop.hive.common.ValidCompactorWriteIdList; +import org.apache.hadoop.hive.common.ValidReaderWriteIdList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.AcidUtils.AcidOperationalProperties; -import org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat; import org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockFile; import org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockFileSystem; import org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockPath; @@ -56,8 +55,8 @@ public class TestAcidUtils { assertEquals("/tmp/000123_0", AcidUtils.createFilename(p, options).toString()); options.bucket(23) - .minimumTransactionId(100) - .maximumTransactionId(200) + .minimumWriteId(100) + .maximumWriteId(200) .writingBase(true) .setOldStyle(false); assertEquals("/tmp/base_0000200/bucket_00023", @@ -92,8 +91,8 @@ public class TestAcidUtils { assertEquals("/tmp/123456789_0", AcidUtils.createFilename(p, options).toString()); options.bucket(23) - .minimumTransactionId(1234567880) - .maximumTransactionId(1234567890) + .minimumWriteId(1234567880) + .maximumWriteId(1234567890) .writingBase(true) .setOldStyle(false); assertEquals("/tmp/base_1234567890/bucket_00023", @@ -118,29 +117,29 @@ public class TestAcidUtils { conf); assertEquals(false, opts.getOldStyle()); assertEquals(true, opts.isWritingBase()); - assertEquals(567, opts.getMaximumTransactionId()); - assertEquals(0, opts.getMinimumTransactionId()); + assertEquals(567, opts.getMaximumWriteId()); + assertEquals(0, opts.getMinimumWriteId()); assertEquals(123, opts.getBucketId()); opts = AcidUtils.parseBaseOrDeltaBucketFilename( new MockPath(fs, dir + "/delta_000005_000006/bucket_00001"), conf); assertEquals(false, opts.getOldStyle()); assertEquals(false, opts.isWritingBase()); - assertEquals(6, opts.getMaximumTransactionId()); - assertEquals(5, opts.getMinimumTransactionId()); + assertEquals(6, opts.getMaximumWriteId()); + assertEquals(5, opts.getMinimumWriteId()); assertEquals(1, opts.getBucketId()); opts = AcidUtils.parseBaseOrDeltaBucketFilename( new MockPath(fs, dir + "/delete_delta_000005_000006/bucket_00001"), conf); assertEquals(false, opts.getOldStyle()); assertEquals(false, opts.isWritingBase()); - assertEquals(6, opts.getMaximumTransactionId()); - assertEquals(5, opts.getMinimumTransactionId()); + assertEquals(6, opts.getMaximumWriteId()); + assertEquals(5, opts.getMinimumWriteId()); assertEquals(1, opts.getBucketId()); opts = AcidUtils.parseBaseOrDeltaBucketFilename(new Path(dir, "000123_0"), conf); assertEquals(true, opts.getOldStyle()); assertEquals(true, opts.isWritingBase()); assertEquals(123, opts.getBucketId()); - assertEquals(0, opts.getMinimumTransactionId()); - assertEquals(0, opts.getMaximumTransactionId()); + assertEquals(0, opts.getMinimumWriteId()); + assertEquals(0, opts.getMaximumWriteId()); } @@ -160,7 +159,7 @@ public class TestAcidUtils { new MockFile("mock:/tbl/part1/subdir/000000_0", 0, new byte[0])); AcidUtils.Directory dir = AcidUtils.getAcidState(new MockPath(fs, "/tbl/part1"), conf, - new ValidReadTxnList("100:" + Long.MAX_VALUE + ":")); + new ValidReaderWriteIdList("tbl:100:" + Long.MAX_VALUE + ":")); assertEquals(null, dir.getBaseDirectory()); assertEquals(0, dir.getCurrentDirectories().size()); assertEquals(0, dir.getObsolete().size()); @@ -195,7 +194,7 @@ public class TestAcidUtils { new MockFile("mock:/tbl/part1/delta_101_101/bucket_0", 0, new byte[0])); AcidUtils.Directory dir = AcidUtils.getAcidState(new MockPath(fs, - "mock:/tbl/part1"), conf, new ValidReadTxnList("100:" + Long.MAX_VALUE + ":")); + "mock:/tbl/part1"), conf, new ValidReaderWriteIdList("tbl:100:" + Long.MAX_VALUE + ":")); assertEquals(null, dir.getBaseDirectory()); List<FileStatus> obsolete = dir.getObsolete(); assertEquals(2, obsolete.size()); @@ -215,12 +214,12 @@ public class TestAcidUtils { assertEquals(2, deltas.size()); AcidUtils.ParsedDelta delt = deltas.get(0); assertEquals("mock:/tbl/part1/delta_025_030", delt.getPath().toString()); - assertEquals(25, delt.getMinTransaction()); - assertEquals(30, delt.getMaxTransaction()); + assertEquals(25, delt.getMinWriteId()); + assertEquals(30, delt.getMaxWriteId()); delt = deltas.get(1); assertEquals("mock:/tbl/part1/delta_050_100", delt.getPath().toString()); - assertEquals(50, delt.getMinTransaction()); - assertEquals(100, delt.getMaxTransaction()); + assertEquals(50, delt.getMinWriteId()); + assertEquals(100, delt.getMaxWriteId()); } @Test @@ -237,7 +236,7 @@ public class TestAcidUtils { new MockFile("mock:/tbl/part1/delta_90_120/bucket_0", 0, new byte[0])); AcidUtils.Directory dir = AcidUtils.getAcidState(new MockPath(fs, - "mock:/tbl/part1"), conf, new ValidReadTxnList("100:" + Long.MAX_VALUE + ":")); + "mock:/tbl/part1"), conf, new ValidReaderWriteIdList("tbl:100:" + Long.MAX_VALUE + ":")); assertEquals("mock:/tbl/part1/base_49", dir.getBaseDirectory().toString()); List<FileStatus> obsolete = dir.getObsolete(); assertEquals(5, obsolete.size()); @@ -251,8 +250,8 @@ public class TestAcidUtils { assertEquals(1, deltas.size()); AcidUtils.ParsedDelta delt = deltas.get(0); assertEquals("mock:/tbl/part1/delta_050_105", delt.getPath().toString()); - assertEquals(50, delt.getMinTransaction()); - assertEquals(105, delt.getMaxTransaction()); + assertEquals(50, delt.getMinWriteId()); + assertEquals(105, delt.getMaxWriteId()); } @Test @@ -265,7 +264,7 @@ public class TestAcidUtils { new MockFile("mock:/tbl/part1/000001_1", 500, new byte[0])); Path part = new MockPath(fs, "/tbl/part1"); AcidUtils.Directory dir = - AcidUtils.getAcidState(part, conf, new ValidReadTxnList("150:" + Long.MAX_VALUE + ":")); + AcidUtils.getAcidState(part, conf, new ValidReaderWriteIdList("tbl:150:" + Long.MAX_VALUE + ":")); // Obsolete list should include the two original bucket files, and the old base dir List<FileStatus> obsolete = dir.getObsolete(); assertEquals(3, obsolete.size()); @@ -286,7 +285,7 @@ public class TestAcidUtils { new MockFile("mock:/tbl/part1/base_50/bucket_0", 500, new byte[0])); Path part = new MockPath(fs, "mock:/tbl/part1"); AcidUtils.Directory dir = - AcidUtils.getAcidState(part, conf, new ValidReadTxnList("100:" + Long.MAX_VALUE + ":")); + AcidUtils.getAcidState(part, conf, new ValidReaderWriteIdList("tbl:100:" + Long.MAX_VALUE + ":")); assertEquals("mock:/tbl/part1/base_50", dir.getBaseDirectory().toString()); List<FileStatus> obsolete = dir.getObsolete(); assertEquals(2, obsolete.size()); @@ -320,8 +319,8 @@ public class TestAcidUtils { new MockFile("mock:/tbl/part1/delta_058_58/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/base_50/bucket_0", 500, new byte[0])); Path part = new MockPath(fs, "mock:/tbl/part1"); - AcidUtils.Directory dir = - AcidUtils.getAcidState(part, conf, new ValidReadTxnList("100:" + Long.MAX_VALUE + ":")); + AcidUtils.Directory dir + = AcidUtils.getAcidState(part, conf, new ValidReaderWriteIdList("tbl:100:" + Long.MAX_VALUE + ":")); assertEquals("mock:/tbl/part1/base_50", dir.getBaseDirectory().toString()); List<FileStatus> obsolete = dir.getObsolete(); assertEquals(5, obsolete.size()); @@ -346,7 +345,7 @@ public class TestAcidUtils { new MockFile("mock:/tbl/part1/delta_1_1/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_2_5/bucket_0", 500, new byte[0])); Path part = new MockPath(fs, "mock:/tbl/part1"); - AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("100:4:4")); + AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidReaderWriteIdList("tbl:100:4:4")); List<AcidUtils.ParsedDelta> delts = dir.getCurrentDirectories(); assertEquals(2, delts.size()); assertEquals("mock:/tbl/part1/delta_1_1", delts.get(0).getPath().toString()); @@ -367,7 +366,7 @@ public class TestAcidUtils { new MockFile("mock:/tbl/part1/delta_4_4_3/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_101_101_1/bucket_0", 500, new byte[0])); Path part = new MockPath(fs, "mock:/tbl/part1"); - AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("100:4:4")); + AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidReaderWriteIdList("tbl:100:4:4")); List<AcidUtils.ParsedDelta> delts = dir.getCurrentDirectories(); assertEquals(2, delts.size()); assertEquals("mock:/tbl/part1/delta_1_1", delts.get(0).getPath().toString()); @@ -382,7 +381,7 @@ public class TestAcidUtils { new MockFile("mock:/tbl/part1/delta_2_5/bucket_0", 500, new byte[0])); Path part = new MockPath(fs, "mock:/tbl/part1"); AcidUtils.Directory dir = - AcidUtils.getAcidState(part, conf, new ValidCompactorTxnList("4:" + Long.MAX_VALUE)); + AcidUtils.getAcidState(part, conf, new ValidCompactorWriteIdList("tbl:4:" + Long.MAX_VALUE)); List<AcidUtils.ParsedDelta> delts = dir.getCurrentDirectories(); assertEquals(1, delts.size()); assertEquals("mock:/tbl/part1/delta_1_1", delts.get(0).getPath().toString()); @@ -399,7 +398,7 @@ public class TestAcidUtils { new MockFile("mock:/tbl/part1/delta_6_10/bucket_0", 500, new byte[0])); Path part = new MockPath(fs, "mock:/tbl/part1"); AcidUtils.Directory dir = - AcidUtils.getAcidState(part, conf, new ValidCompactorTxnList("3:" + Long.MAX_VALUE)); + AcidUtils.getAcidState(part, conf, new ValidCompactorWriteIdList("tbl:3:" + Long.MAX_VALUE)); List<AcidUtils.ParsedDelta> delts = dir.getCurrentDirectories(); assertEquals(1, delts.size()); assertEquals("mock:/tbl/part1/delta_1_1", delts.get(0).getPath().toString()); @@ -424,7 +423,7 @@ public class TestAcidUtils { new MockFile("mock:/tbl/part1/delete_delta_110_110/bucket_0", 0, new byte[0])); AcidUtils.Directory dir = AcidUtils.getAcidState(new MockPath(fs, - "mock:/tbl/part1"), conf, new ValidReadTxnList("100:" + Long.MAX_VALUE + ":")); + "mock:/tbl/part1"), conf, new ValidReaderWriteIdList("tbl:100:" + Long.MAX_VALUE + ":")); assertEquals("mock:/tbl/part1/base_49", dir.getBaseDirectory().toString()); List<FileStatus> obsolete = dir.getObsolete(); assertEquals(7, obsolete.size()); @@ -461,7 +460,7 @@ public class TestAcidUtils { new MockFile("mock:/tbl/part1/base_50/bucket_0", 500, new byte[0])); Path part = new MockPath(fs, "mock:/tbl/part1"); AcidUtils.Directory dir = - AcidUtils.getAcidState(part, conf, new ValidReadTxnList("100:" + Long.MAX_VALUE + ":")); + AcidUtils.getAcidState(part, conf, new ValidReaderWriteIdList("tbl:100:" + Long.MAX_VALUE + ":")); assertEquals("mock:/tbl/part1/base_50", dir.getBaseDirectory().toString()); List<FileStatus> obsolete = dir.getObsolete(); assertEquals(3, obsolete.size()); @@ -490,7 +489,7 @@ public class TestAcidUtils { new MockFile("mock:/tbl/part1/delete_delta_50_50/bucket_0", 500, new byte[0])); Path part = new MockPath(fs, "mock:/tbl/part1"); AcidUtils.Directory dir = - AcidUtils.getAcidState(part, conf, new ValidReadTxnList("100:" + Long.MAX_VALUE + ":")); + AcidUtils.getAcidState(part, conf, new ValidReaderWriteIdList("tbl:100:" + Long.MAX_VALUE + ":")); List<FileStatus> obsolete = dir.getObsolete(); assertEquals(1, obsolete.size()); assertEquals("mock:/tbl/part1/delete_delta_50_50", obsolete.get(0).getPath().toString()); @@ -517,7 +516,7 @@ public class TestAcidUtils { new MockFile("mock:/tbl/part1/delta_6_10/bucket_0", 500, new byte[0])); Path part = new MockPath(fs, "mock:/tbl/part1"); AcidUtils.Directory dir = - AcidUtils.getAcidState(part, conf, new ValidCompactorTxnList("4:" + Long.MAX_VALUE + ":")); + AcidUtils.getAcidState(part, conf, new ValidCompactorWriteIdList("tbl:4:" + Long.MAX_VALUE + ":")); List<AcidUtils.ParsedDelta> delts = dir.getCurrentDirectories(); assertEquals(2, delts.size()); assertEquals("mock:/tbl/part1/delta_1_1", delts.get(0).getPath().toString()); @@ -538,7 +537,7 @@ public class TestAcidUtils { new MockFile("mock:/tbl/part1/delta_4_4_3/bucket_0", 500, new byte[0]), new MockFile("mock:/tbl/part1/delta_101_101_1/bucket_0", 500, new byte[0])); Path part = new MockPath(fs, "mock:/tbl/part1"); - AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidReadTxnList("100:4:4")); + AcidUtils.Directory dir = AcidUtils.getAcidState(part, conf, new ValidReaderWriteIdList("tbl:100:4:4")); List<AcidUtils.ParsedDelta> delts = dir.getCurrentDirectories(); assertEquals(3, delts.size()); assertEquals("mock:/tbl/part1/delta_1_1", delts.get(0).getPath().toString()); http://git-wip-us.apache.org/repos/asf/hive/blob/cbb9233a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java index 8a6a056..ff76002 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java @@ -48,7 +48,7 @@ import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hive.common.ValidTxnList; +import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -2267,7 +2267,7 @@ public class TestInputOutputFormat { // write the orc file to the mock file system Path partDir = new Path(conf.get("mapred.input.dir")); OrcRecordUpdater writer = new OrcRecordUpdater(partDir, - new AcidOutputFormat.Options(conf).maximumTransactionId(10) + new AcidOutputFormat.Options(conf).maximumWriteId(10) .writingBase(true).bucket(0).inspector(inspector).finalDestination(partDir)); for (int i = 0; i < 100; ++i) { BigRow row = new BigRow(i); @@ -2424,7 +2424,7 @@ public class TestInputOutputFormat { // write a base file in partition 0 OrcRecordUpdater writer = new OrcRecordUpdater(partDir[0], - new AcidOutputFormat.Options(conf).maximumTransactionId(10) + new AcidOutputFormat.Options(conf).maximumWriteId(10) .writingBase(true).bucket(0).inspector(inspector).finalDestination(partDir[0])); for(int i=0; i < 10; ++i) { writer.insert(10, new MyRow(i, 2 * i)); @@ -2437,7 +2437,7 @@ public class TestInputOutputFormat { // write a delta file in partition 0 writer = new OrcRecordUpdater(partDir[0], - new AcidOutputFormat.Options(conf).maximumTransactionId(10) + new AcidOutputFormat.Options(conf).maximumWriteId(10) .writingBase(true).bucket(1).inspector(inspector).finalDestination(partDir[0])); for(int i=10; i < 20; ++i) { writer.insert(10, new MyRow(i, 2*i)); @@ -3558,12 +3558,12 @@ public class TestInputOutputFormat { } writer.close(); - AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf).bucket(1).minimumTransactionId(1) - .maximumTransactionId(1).inspector(inspector).finalDestination(mockPath); + AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf).bucket(1).minimumWriteId(1) + .maximumWriteId(1).inspector(inspector).finalDestination(mockPath); OrcOutputFormat of = new OrcOutputFormat(); RecordUpdater ru = of.getRecordUpdater(mockPath, options); for (int i = 0; i < 10; ++i) { - ru.insert(options.getMinimumTransactionId(), new MyRow(i, 2 * i)); + ru.insert(options.getMinimumWriteId(), new MyRow(i, 2 * i)); } ru.close(false);//this deletes the side file @@ -3638,12 +3638,12 @@ public class TestInputOutputFormat { } writer.close(); - AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf).bucket(1).minimumTransactionId(1) - .maximumTransactionId(1).inspector(inspector).finalDestination(mockPath); + AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf).bucket(1).minimumWriteId(1) + .maximumWriteId(1).inspector(inspector).finalDestination(mockPath); OrcOutputFormat of = new OrcOutputFormat(); RecordUpdater ru = of.getRecordUpdater(mockPath, options); for (int i = 0; i < 10; ++i) { - ru.insert(options.getMinimumTransactionId(), new MyRow(i, 2 * i)); + ru.insert(options.getMinimumWriteId(), new MyRow(i, 2 * i)); } ru.close(false);//this deletes the side file @@ -3895,7 +3895,7 @@ public class TestInputOutputFormat { long fileLength = fs.getFileStatus(testFilePath).getLen(); // test with same schema with include - conf.set(ValidTxnList.VALID_TXNS_KEY, "100:99:"); + conf.set(ValidWriteIdList.VALID_WRITEIDS_KEY, "tbl:100:99:"); conf.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS, "a,b,d"); conf.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES, "int,struct<c:int>,string"); conf.set(ColumnProjectionUtils.READ_ALL_COLUMNS, "false"); @@ -3912,7 +3912,7 @@ public class TestInputOutputFormat { while (reader.next(id, struct)) { assertEquals("id " + record, record, id.getRowId()); assertEquals("bucket " + record, 0, id.getBucketProperty()); - assertEquals("trans " + record, 1, id.getTransactionId()); + assertEquals("writeid " + record, 1, id.getWriteId()); assertEquals("a " + record, 42 * record, ((IntWritable) struct.getFieldValue(0)).get()); assertEquals(null, struct.getFieldValue(1)); @@ -3939,7 +3939,7 @@ public class TestInputOutputFormat { while (reader.next(id, struct)) { assertEquals("id " + record, record, id.getRowId()); assertEquals("bucket " + record, 0, id.getBucketProperty()); - assertEquals("trans " + record, 1, id.getTransactionId()); + assertEquals("writeid " + record, 1, id.getWriteId()); assertEquals("a " + record, 42 * record, ((IntWritable) struct.getFieldValue(0)).get()); assertEquals(null, struct.getFieldValue(1)); http://git-wip-us.apache.org/repos/asf/hive/blob/cbb9233a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java index bbd040a..a47979b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java @@ -35,8 +35,8 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.common.ValidReadTxnList; +import org.apache.hadoop.hive.common.ValidReaderWriteIdList; +import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.io.AcidOutputFormat; import org.apache.hadoop.hive.ql.io.AcidUtils; @@ -121,11 +121,11 @@ public class TestOrcRawRecordMerger { long currentTransaction, String value) { event.setFieldValue(OrcRecordUpdater.OPERATION, new IntWritable(operation)); - event.setFieldValue(OrcRecordUpdater.ORIGINAL_TRANSACTION, + event.setFieldValue(OrcRecordUpdater.ORIGINAL_WRITEID, new LongWritable(originalTransaction)); event.setFieldValue(OrcRecordUpdater.BUCKET, new IntWritable(bucket)); event.setFieldValue(OrcRecordUpdater.ROW_ID, new LongWritable(rowId)); - event.setFieldValue(OrcRecordUpdater.CURRENT_TRANSACTION, + event.setFieldValue(OrcRecordUpdater.CURRENT_WRITEID, new LongWritable(currentTransaction)); OrcStruct row = new OrcStruct(1); row.setFieldValue(0, new Text(value)); @@ -195,17 +195,17 @@ public class TestOrcRawRecordMerger { ReaderPair pair = new OrcRawRecordMerger.ReaderPairAcid(key, reader, minKey, maxKey, new Reader.Options(), 0); RecordReader recordReader = pair.getRecordReader(); - assertEquals(10, key.getTransactionId()); + assertEquals(10, key.getWriteId()); assertEquals(20, key.getBucketProperty()); assertEquals(40, key.getRowId()); - assertEquals(120, key.getCurrentTransactionId()); + assertEquals(120, key.getCurrentWriteId()); assertEquals("third", value(pair.nextRecord())); pair.next(pair.nextRecord()); - assertEquals(40, key.getTransactionId()); + assertEquals(40, key.getWriteId()); assertEquals(50, key.getBucketProperty()); assertEquals(60, key.getRowId()); - assertEquals(130, key.getCurrentTransactionId()); + assertEquals(130, key.getCurrentWriteId()); assertEquals("fourth", value(pair.nextRecord())); pair.next(pair.nextRecord()); @@ -221,38 +221,38 @@ public class TestOrcRawRecordMerger { ReaderPair pair = new OrcRawRecordMerger.ReaderPairAcid(key, reader, null, null, new Reader.Options(), 0); RecordReader recordReader = pair.getRecordReader(); - assertEquals(10, key.getTransactionId()); + assertEquals(10, key.getWriteId()); assertEquals(20, key.getBucketProperty()); assertEquals(20, key.getRowId()); - assertEquals(100, key.getCurrentTransactionId()); + assertEquals(100, key.getCurrentWriteId()); assertEquals("first", value(pair.nextRecord())); pair.next(pair.nextRecord()); - assertEquals(10, key.getTransactionId()); + assertEquals(10, key.getWriteId()); assertEquals(20, key.getBucketProperty()); assertEquals(30, key.getRowId()); - assertEquals(110, key.getCurrentTransactionId()); + assertEquals(110, key.getCurrentWriteId()); assertEquals("second", value(pair.nextRecord())); pair.next(pair.nextRecord()); - assertEquals(10, key.getTransactionId()); + assertEquals(10, key.getWriteId()); assertEquals(20, key.getBucketProperty()); assertEquals(40, key.getRowId()); - assertEquals(120, key.getCurrentTransactionId()); + assertEquals(120, key.getCurrentWriteId()); assertEquals("third", value(pair.nextRecord())); pair.next(pair.nextRecord()); - assertEquals(40, key.getTransactionId()); + assertEquals(40, key.getWriteId()); assertEquals(50, key.getBucketProperty()); assertEquals(60, key.getRowId()); - assertEquals(130, key.getCurrentTransactionId()); + assertEquals(130, key.getCurrentWriteId()); assertEquals("fourth", value(pair.nextRecord())); pair.next(pair.nextRecord()); - assertEquals(40, key.getTransactionId()); + assertEquals(40, key.getWriteId()); assertEquals(50, key.getBucketProperty()); assertEquals(61, key.getRowId()); - assertEquals(140, key.getCurrentTransactionId()); + assertEquals(140, key.getCurrentWriteId()); assertEquals("fifth", value(pair.nextRecord())); pair.next(pair.nextRecord()); @@ -303,19 +303,20 @@ public class TestOrcRawRecordMerger { fs.makeQualified(root); fs.create(root); ReaderPair pair = new OrcRawRecordMerger.OriginalReaderPairToRead(key, reader, BUCKET, minKey, maxKey, - new Reader.Options().include(includes), new OrcRawRecordMerger.Options().rootPath(root), conf, new ValidReadTxnList(), 0); + new Reader.Options().include(includes), new OrcRawRecordMerger.Options().rootPath(root), + conf, new ValidReaderWriteIdList(), 0); RecordReader recordReader = pair.getRecordReader(); - assertEquals(0, key.getTransactionId()); + assertEquals(0, key.getWriteId()); assertEquals(bucketProperty, key.getBucketProperty()); assertEquals(2, key.getRowId()); - assertEquals(0, key.getCurrentTransactionId()); + assertEquals(0, key.getCurrentWriteId()); assertEquals("third", value(pair.nextRecord())); pair.next(pair.nextRecord()); - assertEquals(0, key.getTransactionId()); + assertEquals(0, key.getWriteId()); assertEquals(bucketProperty, key.getBucketProperty()); assertEquals(3, key.getRowId()); - assertEquals(0, key.getCurrentTransactionId()); + assertEquals(0, key.getCurrentWriteId()); assertEquals("fourth", value(pair.nextRecord())); pair.next(pair.nextRecord()); @@ -323,8 +324,8 @@ public class TestOrcRawRecordMerger { Mockito.verify(recordReader).close(); } - private static ValidTxnList createMaximalTxnList() { - return new ValidReadTxnList(); + private static ValidWriteIdList createMaximalTxnList() { + return new ValidReaderWriteIdList(); } @Test @@ -339,40 +340,40 @@ public class TestOrcRawRecordMerger { fs.makeQualified(root); fs.create(root); ReaderPair pair = new OrcRawRecordMerger.OriginalReaderPairToRead(key, reader, BUCKET, null, null, - new Reader.Options(), new OrcRawRecordMerger.Options().rootPath(root), conf, new ValidReadTxnList(), 0); + new Reader.Options(), new OrcRawRecordMerger.Options().rootPath(root), conf, new ValidReaderWriteIdList(), 0); assertEquals("first", value(pair.nextRecord())); - assertEquals(0, key.getTransactionId()); + assertEquals(0, key.getWriteId()); assertEquals(bucketProperty, key.getBucketProperty()); assertEquals(0, key.getRowId()); - assertEquals(0, key.getCurrentTransactionId()); + assertEquals(0, key.getCurrentWriteId()); pair.next(pair.nextRecord()); assertEquals("second", value(pair.nextRecord())); - assertEquals(0, key.getTransactionId()); + assertEquals(0, key.getWriteId()); assertEquals(bucketProperty, key.getBucketProperty()); assertEquals(1, key.getRowId()); - assertEquals(0, key.getCurrentTransactionId()); + assertEquals(0, key.getCurrentWriteId()); pair.next(pair.nextRecord()); assertEquals("third", value(pair.nextRecord())); - assertEquals(0, key.getTransactionId()); + assertEquals(0, key.getWriteId()); assertEquals(bucketProperty, key.getBucketProperty()); assertEquals(2, key.getRowId()); - assertEquals(0, key.getCurrentTransactionId()); + assertEquals(0, key.getCurrentWriteId()); pair.next(pair.nextRecord()); assertEquals("fourth", value(pair.nextRecord())); - assertEquals(0, key.getTransactionId()); + assertEquals(0, key.getWriteId()); assertEquals(bucketProperty, key.getBucketProperty()); assertEquals(3, key.getRowId()); - assertEquals(0, key.getCurrentTransactionId()); + assertEquals(0, key.getCurrentWriteId()); pair.next(pair.nextRecord()); assertEquals("fifth", value(pair.nextRecord())); - assertEquals(0, key.getTransactionId()); + assertEquals(0, key.getWriteId()); assertEquals(bucketProperty, key.getBucketProperty()); assertEquals(4, key.getRowId()); - assertEquals(0, key.getCurrentTransactionId()); + assertEquals(0, key.getCurrentWriteId()); pair.next(pair.nextRecord()); assertEquals(null, pair.nextRecord()); @@ -452,13 +453,13 @@ public class TestOrcRawRecordMerger { OrcStruct event = merger.createValue(); assertEquals(true, merger.next(id, event)); - assertEquals(10, id.getTransactionId()); + assertEquals(10, id.getWriteId()); assertEquals(20, id.getBucketProperty()); assertEquals(40, id.getRowId()); assertEquals("third", getValue(event)); assertEquals(true, merger.next(id, event)); - assertEquals(40, id.getTransactionId()); + assertEquals(40, id.getWriteId()); assertEquals(50, id.getBucketProperty()); assertEquals(60, id.getRowId()); assertEquals("fourth", getValue(event)); @@ -477,9 +478,9 @@ public class TestOrcRawRecordMerger { assertEquals("operation", fields.get(OrcRecordUpdater.OPERATION).getFieldName()); assertEquals("currentTransaction", - fields.get(OrcRecordUpdater.CURRENT_TRANSACTION).getFieldName()); + fields.get(OrcRecordUpdater.CURRENT_WRITEID).getFieldName()); assertEquals("originalTransaction", - fields.get(OrcRecordUpdater.ORIGINAL_TRANSACTION).getFieldName()); + fields.get(OrcRecordUpdater.ORIGINAL_WRITEID).getFieldName()); assertEquals("bucket", fields.get(OrcRecordUpdater.BUCKET).getFieldName()); assertEquals("rowId", @@ -538,15 +539,15 @@ public class TestOrcRawRecordMerger { } /*create delta_1_1_0/bucket0 with 1 row and close the file*/ AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf) - .inspector(inspector).bucket(BUCKET).writingBase(false).minimumTransactionId(1) - .maximumTransactionId(1).finalDestination(root); + .inspector(inspector).bucket(BUCKET).writingBase(false).minimumWriteId(1) + .maximumWriteId(1).finalDestination(root); Path delta1_1_0 = new Path(root, AcidUtils.deltaSubdir( - options.getMinimumTransactionId(), options.getMaximumTransactionId(), options.getStatementId())); + options.getMinimumWriteId(), options.getMaximumWriteId(), options.getStatementId())); Path bucket0 = AcidUtils.createBucketFile(delta1_1_0, BUCKET); Path bucket0SideFile = OrcAcidUtils.getSideFile(bucket0); RecordUpdater ru = of.getRecordUpdater(root, options); - ru.insert(options.getMaximumTransactionId(), new MyRow("first")); + ru.insert(options.getMaximumWriteId(), new MyRow("first")); ru.close(false); FileStatus bucket0File = fs.getFileStatus(bucket0); @@ -581,7 +582,7 @@ public class TestOrcRawRecordMerger { // write the empty base AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf) .inspector(inspector).bucket(BUCKET).writingBase(true) - .maximumTransactionId(100).finalDestination(root); + .maximumWriteId(100).finalDestination(root); of.getRecordUpdater(root, options).close(false); { /*OrcRecordUpdater is inconsistent about when it creates empty files and when it does not. @@ -593,8 +594,8 @@ public class TestOrcRawRecordMerger { AcidUtils.baseDir(100)), BUCKET), wo); w.close(); } - ValidTxnList txnList = new ValidReadTxnList("200:" + Long.MAX_VALUE); - AcidUtils.Directory directory = AcidUtils.getAcidState(root, conf, txnList); + ValidWriteIdList writeIdList = new ValidReaderWriteIdList("testEmpty:200:" + Long.MAX_VALUE); + AcidUtils.Directory directory = AcidUtils.getAcidState(root, conf, writeIdList); Path basePath = AcidUtils.createBucketFile(directory.getBaseDirectory(), BUCKET); @@ -646,7 +647,7 @@ public class TestOrcRawRecordMerger { options.statementId(-1); } RecordUpdater ru = of.getRecordUpdater(root, - options.writingBase(true).maximumTransactionId(100)); + options.writingBase(true).maximumWriteId(100)); for(String v: values) { ru.insert(0, new MyRow(v)); } @@ -654,7 +655,7 @@ public class TestOrcRawRecordMerger { // write a delta ru = of.getRecordUpdater(root, options.writingBase(false) - .minimumTransactionId(200).maximumTransactionId(200).recordIdColumn(1)); + .minimumWriteId(200).maximumWriteId(200).recordIdColumn(1)); ru.update(200, new MyRow("update 1", 0, 0, BUCKET_PROPERTY)); ru.update(200, new MyRow("update 2", 2, 0, BUCKET_PROPERTY)); ru.update(200, new MyRow("update 3", 3, 0, BUCKET_PROPERTY)); @@ -662,8 +663,8 @@ public class TestOrcRawRecordMerger { ru.delete(200, new MyRow("", 8, 0, BUCKET_PROPERTY)); ru.close(false); - ValidTxnList txnList = new ValidReadTxnList("200:" + Long.MAX_VALUE); - AcidUtils.Directory directory = AcidUtils.getAcidState(root, conf, txnList); + ValidWriteIdList writeIdList = new ValidReaderWriteIdList("testNewBaseAndDelta:200:" + Long.MAX_VALUE); + AcidUtils.Directory directory = AcidUtils.getAcidState(root, conf, writeIdList); assertEquals(new Path(root, "base_0000100"), directory.getBaseDirectory()); assertEquals(new Path(root, use130Format ? @@ -978,13 +979,13 @@ public class TestOrcRawRecordMerger { merger.close(); // try ignoring the 200 transaction and make sure it works still - ValidTxnList txns = new ValidReadTxnList("2000:200:200"); + ValidWriteIdList writeIds = new ValidReaderWriteIdList("testNewBaseAndDelta:2000:200:200"); //again 1st split is for base/ baseReader = OrcFile.createReader(basePath, OrcFile.readerOptions(conf)); merger = new OrcRawRecordMerger(conf, false, baseReader, false, BUCKET, - txns, new Reader.Options(), + writeIds, new Reader.Options(), new Path[] {deleteDeltaDir}, new OrcRawRecordMerger.Options().isCompacting(false)); assertEquals(null, merger.getMinKey()); @@ -1006,7 +1007,7 @@ public class TestOrcRawRecordMerger { OrcFile.readerOptions(conf)); merger = new OrcRawRecordMerger(conf, false, baseReader, false, BUCKET, - txns, new Reader.Options(), + writeIds, new Reader.Options(), new Path[] {deleteDeltaDir}, new OrcRawRecordMerger.Options().isCompacting(false)); assertEquals(null, merger.getMinKey()); @@ -1101,7 +1102,7 @@ public class TestOrcRawRecordMerger { // write a delta AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf) - .writingBase(false).minimumTransactionId(1).maximumTransactionId(1) + .writingBase(false).minimumWriteId(1).maximumWriteId(1) .bucket(BUCKET).inspector(inspector).filesystem(fs).recordIdColumn(5) .finalDestination(root); @@ -1119,7 +1120,7 @@ public class TestOrcRawRecordMerger { ru.close(false);//this doesn't create a key index presumably because writerOptions are not set on 'options' // write a delta - options = options.minimumTransactionId(100).maximumTransactionId(100); + options = options.minimumWriteId(100).maximumWriteId(100); ru = of.getRecordUpdater(root, options); values = new String[]{null, null, "1.0", null, null, null, null, "3.1"}; for(int i=0; i < values.length - 1; ++i) { @@ -1222,7 +1223,7 @@ public class TestOrcRawRecordMerger { // make 5 stripes with 2 rows each OrcRecordUpdater.OrcOptions options = (OrcRecordUpdater.OrcOptions) new OrcRecordUpdater.OrcOptions(conf) - .writingBase(true).minimumTransactionId(0).maximumTransactionId(0) + .writingBase(true).minimumWriteId(0).maximumWriteId(0) .bucket(BUCKET).inspector(inspector).filesystem(fs); final int BUCKET_PROPERTY = BucketCodec.V1.encode(options); @@ -1240,7 +1241,7 @@ public class TestOrcRawRecordMerger { ru.close(false); // write a delta - options.writingBase(false).minimumTransactionId(1).maximumTransactionId(1) + options.writingBase(false).minimumWriteId(1).maximumWriteId(1) .recordIdColumn(5); ru = of.getRecordUpdater(root, options); values = new String[]{"0.0", null, null, "1.1", null, null, null, @@ -1254,7 +1255,7 @@ public class TestOrcRawRecordMerger { ru.close(false); // write a delta - options.minimumTransactionId(100).maximumTransactionId(100); + options.minimumWriteId(100).maximumWriteId(100); ru = of.getRecordUpdater(root, options); values = new String[]{null, null, "1.0", null, null, null, null, "3.1"}; for(int i=0; i < values.length - 1; ++i) { @@ -1358,7 +1359,7 @@ public class TestOrcRawRecordMerger { AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf) .bucket(BUCKET).inspector(inspector).filesystem(fs) - .writingBase(false).minimumTransactionId(1).maximumTransactionId(1) + .writingBase(false).minimumWriteId(1).maximumWriteId(1) .finalDestination(root); RecordUpdater ru = of.getRecordUpdater(root, options); String[][] values = {new String[]{"a", "b", "c", "d", "e"}, new String[]{"f", "g", "h", "i", "j"}}; @@ -1368,7 +1369,7 @@ public class TestOrcRawRecordMerger { ru.close(false); // write a delta - options.minimumTransactionId(2).maximumTransactionId(2); + options.minimumWriteId(2).maximumWriteId(2); ru = of.getRecordUpdater(root, options); for(int i=0; i < values[1].length; ++i) { ru.insert(2, new MyRow(values[1][i])); @@ -1432,7 +1433,7 @@ public class TestOrcRawRecordMerger { // write a base AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf) - .writingBase(true).minimumTransactionId(0).maximumTransactionId(0) + .writingBase(true).minimumWriteId(0).maximumWriteId(0) .bucket(BUCKET).inspector(inspector).filesystem(fs).finalDestination(root); if(!use130Format) { options.statementId(-1); @@ -1445,8 +1446,8 @@ public class TestOrcRawRecordMerger { ru.close(false); // write a delta - options.writingBase(false).minimumTransactionId(10) - .maximumTransactionId(19); + options.writingBase(false).minimumWriteId(10) + .maximumWriteId(19); ru = of.getRecordUpdater(root, options); values = new String[]{"6", "7", "8"}; for(int i=0; i < values.length; ++i) { http://git-wip-us.apache.org/repos/asf/hive/blob/cbb9233a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java index 7914f0c..ef6dbbb 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRecordUpdater.java @@ -53,9 +53,9 @@ public class TestOrcRecordUpdater { OrcStruct event = new OrcStruct(OrcRecordUpdater.FIELDS); event.setFieldValue(OrcRecordUpdater.OPERATION, new IntWritable(OrcRecordUpdater.INSERT_OPERATION)); - event.setFieldValue(OrcRecordUpdater.CURRENT_TRANSACTION, + event.setFieldValue(OrcRecordUpdater.CURRENT_WRITEID, new LongWritable(100)); - event.setFieldValue(OrcRecordUpdater.ORIGINAL_TRANSACTION, + event.setFieldValue(OrcRecordUpdater.ORIGINAL_WRITEID, new LongWritable(50)); event.setFieldValue(OrcRecordUpdater.BUCKET, new IntWritable(200)); event.setFieldValue(OrcRecordUpdater.ROW_ID, new LongWritable(300)); @@ -101,8 +101,8 @@ public class TestOrcRecordUpdater { .filesystem(fs) .bucket(10) .writingBase(false) - .minimumTransactionId(10) - .maximumTransactionId(19) + .minimumWriteId(10) + .maximumWriteId(19) .inspector(inspector) .reporter(Reporter.NULL) .finalDestination(root); @@ -210,8 +210,8 @@ public class TestOrcRecordUpdater { .filesystem(fs) .bucket(10) .writingBase(false) - .minimumTransactionId(10) - .maximumTransactionId(19) + .minimumWriteId(10) + .maximumWriteId(19) .inspector(inspector) .reporter(Reporter.NULL) .finalDestination(root) @@ -252,8 +252,8 @@ public class TestOrcRecordUpdater { .filesystem(fs) .bucket(bucket) .writingBase(false) - .minimumTransactionId(100) - .maximumTransactionId(100) + .minimumWriteId(100) + .maximumWriteId(100) .inspector(inspector) .reporter(Reporter.NULL) .recordIdColumn(1)
