This is an automated email from the ASF dual-hosted git repository. mahesh pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new e2a8375 HIVE-21958 : The list of table expression in the inclusion and exclusion list should be separated by '|' instead of comma. (Mahesh Kumar Behera reviewed by Sankar Hariappan) e2a8375 is described below commit e2a8375fe3e8e3c601c770556b104bb47debd33e Author: mbehera <mah...@apache.org> AuthorDate: Wed Jul 10 09:09:20 2019 +0530 HIVE-21958 : The list of table expression in the inclusion and exclusion list should be separated by '|' instead of comma. (Mahesh Kumar Behera reviewed by Sankar Hariappan) --- .../parse/TestReplicationWithTableMigrationEx.java | 4 +- .../parse/TestTableLevelReplicationScenarios.java | 100 +++++++++++---------- .../plugin/TestHiveAuthorizerCheckInvocation.java | 2 +- .../org/apache/hadoop/hive/ql/parse/HiveParser.g | 18 +--- .../hive/ql/parse/ReplicationSemanticAnalyzer.java | 25 ++---- .../hive/ql/parse/repl/load/DumpMetaData.java | 68 +++++--------- .../ql/parse/TestReplicationSemanticAnalyzer.java | 20 ++--- .../apache/hadoop/hive/common/repl/ReplScope.java | 55 +++++------- 8 files changed, 114 insertions(+), 178 deletions(-) diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java index 3a86cb7..f0f1ce5 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java @@ -221,14 +221,14 @@ public class TestReplicationWithTableMigrationEx { WarehouseInstance.Tuple tuple = primary .run("use " + primaryDbName) .run("create table t1 (i int, j int)") - .dump(primaryDbName+".['t1']", null); + .dump(primaryDbName+".'t1'", null); replica.run("create database " + replicatedDbName); replica.loadWithoutExplain(replicatedDbName, tuple.dumpLocation); assertTrue(ReplUtils.isFirstIncPending(replica.getDatabase(replicatedDbName).getParameters())); tuple = primary.run("use " + primaryDbName) .run("insert into t1 values (1, 2)") - .dump(primaryDbName+".['t1']", tuple.lastReplicationId); + .dump(primaryDbName+".'t1'", tuple.lastReplicationId); replica.loadWithoutExplain(replicatedDbName, tuple.dumpLocation); assertFalse(ReplUtils.isFirstIncPending(replica.getDatabase(replicatedDbName).getParameters())); } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java index a3f56f7..09db38d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java @@ -204,7 +204,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios createTables(originalMMAcidTables, CreateTableType.MM_ACID); // Replicate and verify if only 2 tables are replicated to target. - String replPolicy = primaryDbName + ".['t1', 't4', 't5']"; + String replPolicy = primaryDbName + ".'(t1)|(t4)|(t5)'"; String[] replicatedTables = new String[] {"t1", "t4", "t5"}; replicateAndVerify(replPolicy, null, null, null, replicatedTables); } @@ -215,7 +215,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios createTables(originalTables, CreateTableType.NON_ACID); // Replicate and verify if only 3 tables are replicated to target. - String replPolicy = primaryDbName + ".['t1*', 't3'].['t100']"; + String replPolicy = primaryDbName + ".'(t1*)|(t3)'.'t100'"; String[] replicatedTables = new String[] {"t1", "t11", "t3"}; replicateAndVerify(replPolicy, null, null, null, replicatedTables); } @@ -234,7 +234,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios createTables(originalMMAcidTables, CreateTableType.MM_ACID); // Replicate and verify if only 2 tables are replicated to target. - String replPolicy = primaryDbName + ".['t1', 't5']"; + String replPolicy = primaryDbName + ".'t1|t5'"; String[] replicatedTables = new String[] {"t1", "t5"}; replicateAndVerify(replPolicy, tupleBootstrap.lastReplicationId, null, null, replicatedTables); } @@ -249,7 +249,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios createTables(originalTables, CreateTableType.NON_ACID); // Replicate and verify if only 3 tables are replicated to target. - String replPolicy = primaryDbName + ".['t1+', 't2'].['t11', 't3']"; + String replPolicy = primaryDbName + ".'(t1+)|(t2)'.'t11|t3'"; String[] replicatedTables = new String[] {"t1", "t111", "t2"}; replicateAndVerify(replPolicy, tupleBootstrap.lastReplicationId, null, null, replicatedTables); } @@ -263,12 +263,13 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios String[] replicatedTables = new String[] {}; boolean failed; String[] invalidReplPolicies = new String[] { - primaryDbName + ".t1.t2", // Two explicit table names not allowed. - primaryDbName + ".['t1'].t2", // Table name and include list not allowed. - primaryDbName + ".t1.['t2']", // Table name and exclude list not allowed. - primaryDbName + ".[t1].t2", // Table name and include list not allowed. - primaryDbName + ".['t1+'].", // Abrubtly ended dot. - primaryDbName + "..[]" // Multiple dots + primaryDbName + ".t1.t2", // Didn't enclose table pattern within single quotes. + primaryDbName + ".'t1'.t2", // Table name and include list not allowed. + primaryDbName + ".t1.'t2'", // Table name and exclude list not allowed. + primaryDbName + ".'t1+'.", // Abrubtly ended dot. + primaryDbName + ".['t1+'].['t11']", // With square brackets + primaryDbName + "..''", // Two dots with empty list + primaryDbName + ".'t1'.'tt2'.'t3'" // More than two list }; for (String replPolicy : invalidReplPolicies) { failed = false; @@ -302,7 +303,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios // Replace with replication policy having different DB name. String oldReplPolicy = replPolicy; - replPolicy = primaryDbName + "_dupe.['t1+'].['t1']"; + replPolicy = primaryDbName + "_dupe.'t1+'.'t1'"; failed = false; try { replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, null, null, null, replicatedTables); @@ -313,18 +314,23 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios } Assert.assertTrue(failed); - // Invalid pattern where we didn't enclose table pattern within single or double quotes. - replPolicy = primaryDbName + ".[t1].[t2]"; - failed = false; - try { - replicateAndVerify(replPolicy, null, null, null, replicatedTables); - } catch (Exception ex) { - LOG.info("Got exception: {}", ex.getMessage()); - Assert.assertTrue(ex instanceof SemanticException); - Assert.assertTrue(ex.getMessage().equals(ErrorMsg.REPL_INVALID_DB_OR_TABLE_PATTERN.getMsg())); - failed = true; + // Invalid pattern, include/exclude table list is empty. + invalidReplPolicies = new String[] { + primaryDbName + ".''.'t2'", // Include list is empty. + primaryDbName + ".'t1'.''" // Exclude list is empty. + }; + for (String invalidReplPolicy : invalidReplPolicies) { + failed = false; + try { + replicateAndVerify(invalidReplPolicy, null, null, null, replicatedTables); + } catch (Exception ex) { + LOG.info("Got exception: {}", ex.getMessage()); + Assert.assertTrue(ex instanceof SemanticException); + Assert.assertTrue(ex.getMessage().equals(ErrorMsg.REPL_INVALID_DB_OR_TABLE_PATTERN.getMsg())); + failed = true; + } + Assert.assertTrue(failed); } - Assert.assertTrue(failed); } @Test @@ -334,8 +340,8 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios // List of repl policy formats that leads to Full DB replication. String[] fullDbReplPolicies = new String[] { - primaryDbName + ".['.*?']", - primaryDbName + ".['.*?'].[]" + primaryDbName + ".'.*?'", + primaryDbName }; // Replicate and verify if all 3 tables are replicated to target. @@ -350,13 +356,13 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios createTables(originalTables, CreateTableType.NON_ACID); // Replicate and verify if 2 tables are replicated as per policy. - String replPolicy = primaryDbName.toUpperCase() + ".['.*a1+', 'cc3', 'B2'].['AA1+', 'b2']"; + String replPolicy = primaryDbName.toUpperCase() + ".'(.*a1+)|(cc3)|(B2)'.'(AA1+)|(b2)'"; String[] replicatedTables = new String[] {"a1", "cc3"}; String lastReplId = replicateAndVerify(replPolicy, null, null, null, replicatedTables); // Test case insensitive nature in REPLACE clause as well. String oldReplPolicy = replPolicy; - replPolicy = primaryDbName + ".['.*a1+', 'cc3', 'B2'].['AA1+']"; + replPolicy = primaryDbName + ".'(.*a1+)|(cc3)|(B2)'.'AA1+'"; replicatedTables = new String[] {"a1", "b2", "cc3"}; String[] bootstrappedTables = new String[] {"b2"}; replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables); @@ -374,7 +380,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios // Replicate and verify if only non-acid tables are replicated to target. List<String> dumpWithoutAcidClause = Collections.singletonList( "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'"); - String replPolicy = primaryDbName + ".['a[0-9]+', 'b1'].['a4']"; + String replPolicy = primaryDbName + ".'(a[0-9]+)|(b1)'.'a4'"; String[] bootstrapReplicatedTables = new String[] {"a1"}; String lastReplId = replicateAndVerify(replPolicy, null, dumpWithoutAcidClause, null, bootstrapReplicatedTables); @@ -398,7 +404,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios List<String> dumpWithClause = Collections.singletonList( "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'" ); - String replPolicy = primaryDbName + ".['a[0-9]+', 'b2'].['a1']"; + String replPolicy = primaryDbName + ".'(a[0-9]+)|(b2)'.'a1'"; String[] replicatedTables = new String[] {"a2", "b2"}; WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName) .dump(replPolicy, null, dumpWithClause); @@ -429,7 +435,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios List<String> dumpWithClause = Collections.singletonList( "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'" ); - String replPolicy = primaryDbName + ".['a[0-9]+', 'b2'].['a1']"; + String replPolicy = primaryDbName + ".'(a[0-9]+)|(b2)'.'a1'"; String[] bootstrapReplicatedTables = new String[] {"b2"}; String lastReplId = replicateAndVerify(replPolicy, null, dumpWithClause, loadWithClause, bootstrapReplicatedTables); @@ -465,7 +471,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios createTables(originalMMAcidTables, CreateTableType.MM_ACID); // Replicate and verify if only 2 tables are replicated to target. - String replPolicy = primaryDbName + ".['t1', 't4']"; + String replPolicy = primaryDbName + ".'(t1)|(t4)'"; String oldReplPolicy = null; String[] replicatedTables = new String[] {"t1", "t4"}; String lastReplId = replicateAndVerify(replPolicy, null, null, null, replicatedTables); @@ -473,7 +479,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios // Exclude t4 and include t3, t6 createTables(new String[] {"t6"}, CreateTableType.MM_ACID); oldReplPolicy = replPolicy; - replPolicy = primaryDbName + ".['t1', 't3', 't6']"; + replPolicy = primaryDbName + ".'t1|t3|t6'"; replicatedTables = new String[] {"t1", "t3", "t6"}; String[] bootstrappedTables = new String[] {"t3", "t6"}; lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, @@ -489,7 +495,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios // Convert to regex that excludes t3, t4 and t5. oldReplPolicy = replPolicy; - replPolicy = primaryDbName + ".['.*?'].['t[3-5]+']"; + replPolicy = primaryDbName + ".'.*?'.'t[3-5]+'"; replicatedTables = new String[] {"t1", "t2", "t6"}; bootstrappedTables = new String[] {}; replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, @@ -508,7 +514,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios // Replicate and verify if only non-acid tables are replicated to target. List<String> dumpWithoutAcidClause = Collections.singletonList( "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'"); - String replPolicy = primaryDbName + ".['a[0-9]+', 'b[0-9]+'].['b1']"; + String replPolicy = primaryDbName + ".'(a[0-9]+)|(b[0-9]+)'.'b1'"; String[] bootstrapReplicatedTables = new String[] {"a1"}; String lastReplId = replicateAndVerify(replPolicy, null, dumpWithoutAcidClause, null, bootstrapReplicatedTables); @@ -516,7 +522,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios // Enable acid tables for replication. Also, replace, replication policy to exclude "b1" and "a3" // instead of "a1" alone. String oldReplPolicy = replPolicy; - replPolicy = primaryDbName + ".['[a-z]+[0-9]+'].['a3', 'b1']"; + replPolicy = primaryDbName + ".'[a-z]+[0-9]+'.'(a3)|(b1)'"; List<String> dumpWithAcidBootstrapClause = Arrays.asList( "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES + "'='true'"); @@ -536,7 +542,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios // Replicate and verify if only non-acid tables are replicated to target. List<String> dumpWithoutAcidClause = Collections.singletonList( "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'"); - String replPolicy = primaryDbName + ".['a[0-9]+', 'b[0-9]+'].['b1']"; + String replPolicy = primaryDbName + ".'(a[0-9]+)|(b[0-9]+)'.'b1'"; String[] bootstrapReplicatedTables = new String[] {"a1"}; String lastReplId = replicateAndVerify(replPolicy, null, dumpWithoutAcidClause, null, bootstrapReplicatedTables); @@ -545,7 +551,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios // "a2" but exclude "a1" and "b1". Still ACID tables shouldn't be bootstrapped. Only non-ACID // table "b1" should be bootstrapped. String oldReplPolicy = replPolicy; - replPolicy = primaryDbName + ".['a[0-9]+', 'b[0-9]+'].['a2']"; + replPolicy = primaryDbName + ".'(a[0-9]+)|(b[0-9]+)'.'a2'"; String[] incrementalReplicatedTables = new String[] {"a1", "b1"}; String[] bootstrappedTables = new String[] {"b1"}; lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, @@ -564,7 +570,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios List<String> dumpWithClause = Collections.singletonList( "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'" ); - String replPolicy = primaryDbName + ".['a[0-9]+', 'b1'].['a1']"; + String replPolicy = primaryDbName + ".'(a[0-9]+)|(b1)'.'a1'"; String[] bootstrapReplicatedTables = new String[] {"b1"}; String lastReplId = replicateAndVerify(replPolicy, null, dumpWithClause, loadWithClause, bootstrapReplicatedTables); @@ -572,7 +578,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios // Continue to disable external tables for replication. Also, replace, replication policy to exclude // "b1" and include "a1". String oldReplPolicy = replPolicy; - replPolicy = primaryDbName + ".['a[0-9]+', 'b[0-9]+'].['a2', 'b1']"; + replPolicy = primaryDbName + ".'(a[0-9]+)|(b[0-9]+)'.'(a2)|(b1)'"; String[] incrementalReplicatedTables = new String[] {"a1"}; String[] bootstrappedTables = new String[] {"a1"}; lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, @@ -581,7 +587,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios // Enable external tables replication and bootstrap in incremental phase. Also, replace, // replication policy to exclude tables with prefix "b". oldReplPolicy = replPolicy; - replPolicy = primaryDbName + ".['[a-z]+[0-9]+'].['b[0-9]+']"; + replPolicy = primaryDbName + ".'[a-z]+[0-9]+'.'b[0-9]+'"; incrementalReplicatedTables = new String[] {"a1", "a2", "c2"}; bootstrappedTables = new String[] {"a2", "c2"}; dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", @@ -608,7 +614,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios @Test public void testRenameTableScenariosBasic() throws Throwable { - String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']"; + String replPolicy = primaryDbName + ".'in[0-9]+'.'out[0-9]+'"; String lastReplId = replicateAndVerify(replPolicy, null, null, null, null, new String[] {}, new String[] {}); @@ -662,7 +668,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios @Test public void testRenameTableScenariosWithDmlOperations() throws Throwable { - String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']"; + String replPolicy = primaryDbName + ".'in[0-9]+'.'out[0-9]+'"; String lastReplId = replicateAndVerify(replPolicy, null, null, null, null, new String[] {}, new String[] {}); @@ -694,7 +700,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios @Test public void testRenameTableScenariosAcidTable() throws Throwable { - String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']"; + String replPolicy = primaryDbName + ".'in[0-9]+'.'out[0-9]+'"; List<String> dumpWithClause = Arrays.asList( "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname + "'='false'", "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'" @@ -731,7 +737,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios @Test public void testRenameTableScenariosExternalTable() throws Throwable { - String replPolicy = primaryDbName + ".['in[0-9]+'].['out[0-9]+']"; + String replPolicy = primaryDbName + ".'in[0-9]+'.'out[0-9]+'"; List<String> loadWithClause = ReplicationTestUtils.externalTableBasePathWithClause(REPLICA_EXTERNAL_BASE, replica); List<String> dumpWithClause = Arrays.asList( "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'", @@ -794,7 +800,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'" ); - String replPolicy = primaryDbName + ".['in[0-9]+', 'out4', 'out5', 'out1500']"; + String replPolicy = primaryDbName + ".'(in[0-9]+)|(out4)|(out5)|(out1500)'"; String lastReplId = replicateAndVerify(replPolicy, null, null, dumpWithClause, loadWithClause, new String[] {}, new String[] {}); @@ -813,7 +819,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios .run("alter table out11 rename to out12") // Old name not matching old and new name not matching both .run("alter table out12 rename to in12"); // Old name not matching old and new name matching both - String newPolicy = primaryDbName + ".['in[0-9]+', 'out1500'].['in2']"; + String newPolicy = primaryDbName + ".'(in[0-9]+)|(out1500)|(in2)'"; dumpWithClause = Arrays.asList( "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='false'" @@ -828,7 +834,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios @Test public void testRenameTableScenariosWithReplacePolicyDMLOperattion() throws Throwable { - String replPolicy = primaryDbName + ".['in[0-9]+', 'out5000', 'out5001'].['in100', 'in200', 'in305']"; + String replPolicy = primaryDbName + ".'(in[0-9]+)|(out5000)|(out5001)'.'(in100)|(in200)|(in305)'"; String lastReplId = replicateAndVerify(replPolicy, null, null, null, null, new String[] {}, new String[] {}); @@ -845,7 +851,7 @@ public class TestTableLevelReplicationScenarios extends BaseReplicationScenarios null, bootstrapTables, replicatedTables); // Rename the tables to satisfy the condition also replace the policy. - String newPolicy = primaryDbName + ".['in[0-9]+', 'out3000'].['in2']"; + String newPolicy = primaryDbName + ".'(in[0-9]+)|(out3000)'.'in2'"; primary.run("use " + primaryDbName) .run("alter table in200 rename to in2000") // Old name not matching old, old and new matching new policy. .run("alter table in400 rename to out400") // Old name matching new and old policy, new matching none. diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java index eeda423..5c803ca 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java @@ -582,7 +582,7 @@ public class TestHiveAuthorizerCheckInvocation { assertEquals("db name", dbName.toLowerCase(), dbObj.getDbname()); resetAuthorizer(); - status = driver.compile("repl dump " + dbName + ".['" + inDbTableName + "']"); + status = driver.compile("repl dump " + dbName + ".'" + inDbTableName + "'"); assertEquals(0, status); inputs = getHivePrivilegeObjectInputs().getLeft(); dbObj = inputs.get(0); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index a4b233a..3a2a2d0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -939,26 +939,10 @@ replTableLevelPolicy @init { pushMsg("Replication table level policy definition", state); } @after { popMsg(state); } : - ((replTablesIncludeList=replTablesList) (DOT replTablesExcludeList=replTablesList)?) + ((replTablesIncludeList=StringLiteral) (DOT replTablesExcludeList=StringLiteral)?) -> ^(TOK_REPL_TABLES $replTablesIncludeList $replTablesExcludeList?) ; -replTablesList -@init { pushMsg("replication table name or comma separated table names pattern list", state); } -@after { popMsg(state); } - : - (LSQUARE (tablePattern (COMMA tablePattern)*)? RSQUARE) -> ^(TOK_REPL_TABLES_LIST tablePattern*) - ; - -tablePattern -@init { pushMsg("Table name pattern", state); } -@after { popMsg(state); } - : - (pattern=StringLiteral) -> $pattern - | - (identifier) -> TOK_NULL - ; - replStatusStatement @init { pushMsg("replication status statement", state); } @after { popMsg(state); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java index df41a2e..39789ca 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java @@ -58,14 +58,12 @@ import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_MOVE_OPTIMIZED_ import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_DBNAME; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_FROM; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_LIMIT; -import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_NULL; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPLACE; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_CONFIG; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_DUMP; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_LOAD; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_STATUS; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_TABLES; -import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_TABLES_LIST; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TO; public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { @@ -140,25 +138,16 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { // and exclude tables lists. String replScopeType = (replScope == this.replScope) ? "Current" : "Old"; for (int listIdx = 0; listIdx < childCount; listIdx++) { - Tree tablesListNode = replTablesNode.getChild(listIdx); - assert(tablesListNode.getType() == TOK_REPL_TABLES_LIST); - - List<String> tablesList = new ArrayList<>(); - for (int child = 0; child < tablesListNode.getChildCount(); child++) { - Tree tablePatternNode = tablesListNode.getChild(child); - if (tablePatternNode.getType() == TOK_NULL) { - LOG.error(ErrorMsg.REPL_INVALID_DB_OR_TABLE_PATTERN.getMsg()); - throw new SemanticException(ErrorMsg.REPL_INVALID_DB_OR_TABLE_PATTERN.getMsg()); - } - tablesList.add(unescapeSQLString(tablePatternNode.getText())); + String tableList = unescapeSQLString(replTablesNode.getChild(listIdx).getText()); + if (tableList == null || tableList.isEmpty()) { + throw new SemanticException(ErrorMsg.REPL_INVALID_DB_OR_TABLE_PATTERN); } - if (listIdx == 0) { - LOG.info("{} ReplScope: Set Included Tables List: {}", replScopeType, tablesList); - replScope.setIncludedTablePatterns(tablesList); + LOG.info("{} ReplScope: Set Included Tables List: {}", replScopeType, tableList); + replScope.setIncludedTablePatterns(tableList); } else { - LOG.info("{} ReplScope: Set Excluded Tables List: {}", replScopeType, tablesList); - replScope.setExcludedTablePatterns(tablesList); + LOG.info("{} ReplScope: Set Excluded Tables List: {}", replScopeType, tableList); + replScope.setExcludedTablePatterns(tableList); } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java index 4c137eb..f275194 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java @@ -85,48 +85,26 @@ public class DumpMetaData { } String[] lineContents = line.split("\t"); - if (lineContents.length < 1) { - return; - } - replScope = new ReplScope(); - - LOG.info("Read ReplScope: Set Db Name: {}.", lineContents[0]); - replScope.setDbName(lineContents[0]); - - // Read/set include and exclude tables list. - int idx = readReplScopeTablesList(lineContents, 1, true); - readReplScopeTablesList(lineContents, idx, false); - } - - private int readReplScopeTablesList(String[] lineContents, int startIdx, boolean includeList) - throws IOException { - // If the list doesn't exist, then return. - if (startIdx >= lineContents.length) { - return startIdx; - } - - // Each tables list should start with "{" and ends with "}" - if (!"{".equals(lineContents[startIdx])) { - throw new IOException("Invalid repl tables list data in dump metadata file. Missing \"{\"."); - } - - List<String>tableNames = new ArrayList<>(); - for (int i = (startIdx + 1); i < lineContents.length; i++) { - String value = lineContents[i]; - if ("}".equals(value)) { - if (includeList) { - LOG.info("Read ReplScope: Set Include Table Names: {}.", tableNames); - replScope.setIncludedTablePatterns(tableNames); - } else { - LOG.info("Read ReplScope: Set Exclude Table Names: {}.", tableNames); - replScope.setExcludedTablePatterns(tableNames); - } - return (i + 1); + for (int idx = 0; idx < lineContents.length; idx++) { + String value = lineContents[idx]; + switch (idx) { + case 0: + LOG.info("Read ReplScope: Set Db Name: {}.", value); + replScope.setDbName(value); + break; + case 1: + LOG.info("Read ReplScope: Include table name list: {}.", value); + replScope.setIncludedTablePatterns(value); + break; + case 2: + LOG.info("Read ReplScope: Exclude table name list: {}.", value); + replScope.setExcludedTablePatterns(value); + break; + default: + throw new IOException("Invalid repl tables list data in dump metadata file"); } - tableNames.add(value); } - throw new IOException("Invalid repl tables list data in dump metadata file. Missing \"}\"."); } private void loadDumpFromFile() throws SemanticException { @@ -205,17 +183,13 @@ public class DumpMetaData { List<String> values = new ArrayList<>(); values.add(replScope.getDbName()); - List<String> includedTableNames = replScope.getIncludedTableNames(); - List<String> excludedTableNames = replScope.getExcludedTableNames(); + String includedTableNames = replScope.getIncludedTableNames(); + String excludedTableNames = replScope.getExcludedTableNames(); if (includedTableNames != null) { - values.add("{"); - values.addAll(includedTableNames); - values.add("}"); + values.add(includedTableNames); } if (excludedTableNames != null) { - values.add("{"); - values.addAll(excludedTableNames); - values.add("}"); + values.add(excludedTableNames); } LOG.info("Preparing ReplScope {} to dump.", values); return values; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java index 9db4e96..48b9883 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java @@ -77,9 +77,7 @@ public class TestReplicationSemanticAnalyzer { ASTNode child = (ASTNode) root.getChild(1); assertEquals("TOK_REPL_TABLES", child.getText()); assertEquals(1, child.getChildCount()); - assertEquals("TOK_REPL_TABLES_LIST", child.getChild(0).getText()); - assertEquals(1, child.getChild(0).getChildCount()); - assertEquals("'test_table'", child.getChild(0).getChild(0).getText()); + assertEquals("'test_table'", child.getChild(0).getText()); } private static void assertDatabase(final int expectedNumberOfChildren, ASTNode root) { @@ -110,14 +108,14 @@ public class TestReplicationSemanticAnalyzer { @Test public void parseTableName() throws ParseException { - ASTNode root = parse("repl dump testDb.['test_table']"); + ASTNode root = parse("repl dump testDb.'test_table'"); assertDatabase(2, root); assertTableName(root); } @Test public void parseFromEventId() throws ParseException { - ASTNode root = parse("repl dump testDb.['test_table'] from 100"); + ASTNode root = parse("repl dump testDb.'test_table' from 100"); assertDatabase(3, root); assertTableName(root); assertFromEvent(1, root); @@ -125,7 +123,7 @@ public class TestReplicationSemanticAnalyzer { @Test public void parseToEventId() throws ParseException { - ASTNode root = parse("repl dump testDb.['test_table'] from 100 to 200"); + ASTNode root = parse("repl dump testDb.'test_table' from 100 to 200"); assertDatabase(3, root); assertTableName(root); ASTNode fromClauseRootNode = assertFromEvent(3, root); @@ -134,7 +132,7 @@ public class TestReplicationSemanticAnalyzer { @Test public void parseLimit() throws ParseException { - ASTNode root = parse("repl dump testDb.['test_table'] from 100 to 200 limit 10"); + ASTNode root = parse("repl dump testDb.'test_table' from 100 to 200 limit 10"); assertDatabase(3, root); assertTableName(root); ASTNode fromClauseRootNode = assertFromEvent(5, root); @@ -162,7 +160,7 @@ public class TestReplicationSemanticAnalyzer { @Test public void parseTableName() throws ParseException { ASTNode root = - parse("repl dump testDb.['test_table'] with ('key.1'='value.1','key.2'='value.2')"); + parse("repl dump testDb.'test_table' with ('key.1'='value.1','key.2'='value.2')"); assertDatabase(3, root); assertTableName(root); assertWithClause(root, 2); @@ -170,7 +168,7 @@ public class TestReplicationSemanticAnalyzer { @Test public void parseFromEventId() throws ParseException { - ASTNode root = parse("repl dump testDb.['test_table'] from 100 " + ASTNode root = parse("repl dump testDb.'test_table' from 100 " + "with ('key.1'='value.1','key.2'='value.2')"); assertDatabase(4, root); assertTableName(root); @@ -180,7 +178,7 @@ public class TestReplicationSemanticAnalyzer { @Test public void parseToEventId() throws ParseException { - ASTNode root = parse("repl dump testDb.['test_table'] from 100 to 200 " + ASTNode root = parse("repl dump testDb.'test_table' from 100 to 200 " + "with ('key.1'='value.1','key.2'='value.2')"); assertDatabase(4, root); assertTableName(root); @@ -191,7 +189,7 @@ public class TestReplicationSemanticAnalyzer { @Test public void parseLimit() throws ParseException { - ASTNode root = parse("repl dump testDb.['test_table'] from 100 to 200 limit 10 " + ASTNode root = parse("repl dump testDb.'test_table' from 100 to 200 limit 10 " + "with ('key.1'='value.1','key.2'='value.2')"); assertDatabase(4, root); assertTableName(root); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java index 14b58a3..8b26f1c 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hive.common.repl; import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; import java.util.regex.Pattern; /** @@ -31,10 +29,10 @@ public class ReplScope implements Serializable { private Pattern dbNamePattern; // Include and exclude table names/patterns exist only for REPL DUMP. - private List<String> includedTableNames; - private List<String> excludedTableNames; - private List<Pattern> includedTableNamePatterns; - private List<Pattern> excludedTableNamePatterns; + private String includedTableNames; + private String excludedTableNames; + private Pattern includedTableNamePattern; + private Pattern excludedTableNamePattern; public ReplScope() { } @@ -53,27 +51,27 @@ public class ReplScope implements Serializable { return dbName; } - public void setIncludedTablePatterns(List<String> includedTableNames) { + public void setIncludedTablePatterns(String includedTableNames) { this.includedTableNames = includedTableNames; - this.includedTableNamePatterns = compilePatterns(includedTableNames); + this.includedTableNamePattern = compilePattern(includedTableNames); } - public List<String> getIncludedTableNames() { + public String getIncludedTableNames() { return includedTableNames; } - public void setExcludedTablePatterns(List<String> excludedTableNames) { + public void setExcludedTablePatterns(String excludedTableNames) { this.excludedTableNames = excludedTableNames; - this.excludedTableNamePatterns = compilePatterns(excludedTableNames); + this.excludedTableNamePattern = compilePattern(excludedTableNames); } - public List<String> getExcludedTableNames() { + public String getExcludedTableNames() { return excludedTableNames; } public boolean includeAllTables() { - return ((includedTableNamePatterns == null) - && (excludedTableNamePatterns == null)); + return ((includedTableNamePattern == null) + && (excludedTableNamePattern == null)); } public boolean includedInReplScope(final String dbName, final String tableName) { @@ -93,42 +91,29 @@ public class ReplScope implements Serializable { return (inTableIncludedList(tableName) && !inTableExcludedList(tableName)); } - private List<Pattern> compilePatterns(List<String> patterns) { - if (patterns == null || patterns.isEmpty()) { + private Pattern compilePattern(String pattern) { + if (pattern == null) { return null; } - List<Pattern> compiledPatterns = new ArrayList<>(); - for (String pattern : patterns) { - // Convert the pattern to lower case because events/HMS will have table names in lower case. - compiledPatterns.add(Pattern.compile(pattern, Pattern.CASE_INSENSITIVE)); - } - return compiledPatterns; - } - private boolean tableMatchAnyPattern(final String tableName, List<Pattern> tableNamePatterns) { - assert(tableNamePatterns != null); - for (Pattern tableNamePattern : tableNamePatterns) { - if (tableNamePattern.matcher(tableName).matches()) { - return true; - } - } - return false; + // Convert the pattern to lower case because events/HMS will have table names in lower case. + return Pattern.compile(pattern, Pattern.CASE_INSENSITIVE); } private boolean inTableIncludedList(final String tableName) { - if (includedTableNamePatterns == null) { + if (includedTableNamePattern == null) { // If included list is empty, repl policy should be full db replication. // So, all tables must be included. return true; } - return tableMatchAnyPattern(tableName, includedTableNamePatterns); + return includedTableNamePattern.matcher(tableName).matches(); } private boolean inTableExcludedList(final String tableName) { - if (excludedTableNamePatterns == null) { + if (excludedTableNamePattern == null) { // If excluded tables list is empty means, all tables in included list must be accepted. return false; } - return tableMatchAnyPattern(tableName, excludedTableNamePatterns); + return excludedTableNamePattern.matcher(tableName).matches(); } }