Repository: hive Updated Branches: refs/heads/master 16b8575b7 -> 637224fcc
HIVE-18626: Repl load "with" clause does not pass config to tasks (Daniel Dai, reviewed by Thejas Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/637224fc Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/637224fc Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/637224fc Branch: refs/heads/master Commit: 637224fcce031bb036718621d05bb714902562a0 Parents: 16b8575 Author: Daniel Dai <da...@hortonworks.com> Authored: Thu Feb 8 00:08:32 2018 -0800 Committer: Daniel Dai <da...@hortonworks.com> Committed: Thu Feb 8 00:08:32 2018 -0800 ---------------------------------------------------------------------- .../hive/ql/exec/repl/bootstrap/load/LoadDatabase.java | 6 +++--- .../ql/exec/repl/bootstrap/load/table/LoadPartitions.java | 10 ++++++---- .../hive/ql/exec/repl/bootstrap/load/table/LoadTable.java | 2 +- .../hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java | 4 ++-- .../org/apache/hadoop/hive/ql/plan/ImportTableDesc.java | 8 ++++---- 5 files changed, 16 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/637224fc/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java index 537c5e9..9191d22 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java @@ -91,14 +91,14 @@ public class LoadDatabase { // db. // TODO: we might revisit this in create-drop-recreate cases, needs some thinking on. DDLWork work = new DDLWork(new HashSet<>(), new HashSet<>(), createDbDesc); - return TaskFactory.get(work, context.hiveConf); + return TaskFactory.get(work, context.hiveConf, true); } private static Task<? extends Serializable> alterDbTask(Database dbObj, HiveConf hiveConf) { AlterDatabaseDesc alterDbDesc = new AlterDatabaseDesc(dbObj.getName(), dbObj.getParameters(), null); DDLWork work = new DDLWork(new HashSet<>(), new HashSet<>(), alterDbDesc); - return TaskFactory.get(work, hiveConf); + return TaskFactory.get(work, hiveConf, true); } private Task<? extends Serializable> setOwnerInfoTask(Database dbObj) { @@ -106,7 +106,7 @@ public class LoadDatabase { new PrincipalDesc(dbObj.getOwnerName(), dbObj.getOwnerType()), null); DDLWork work = new DDLWork(new HashSet<>(), new HashSet<>(), alterDbDesc); - return TaskFactory.get(work, context.hiveConf); + return TaskFactory.get(work, context.hiveConf, true); } private boolean existEmptyDb(String dbName) throws InvalidOperationException, HiveException { http://git-wip-us.apache.org/repos/asf/hive/blob/637224fc/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java index bad7962..0a82225 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java @@ -113,7 +113,7 @@ public class LoadPartitions { private void createTableReplLogTask() throws SemanticException { ReplStateLogWork replLogWork = new ReplStateLogWork(replLogger, tableDesc.getTableName(), tableDesc.tableType()); - Task<ReplStateLogWork> replLogTask = TaskFactory.get(replLogWork, context.hiveConf); + Task<ReplStateLogWork> replLogTask = TaskFactory.get(replLogWork, context.hiveConf, true); if (tracker.tasks().isEmpty()) { tracker.addTask(replLogTask); @@ -224,7 +224,8 @@ public class LoadPartitions { Task<?> addPartTask = TaskFactory.get( new DDLWork(new HashSet<>(), new HashSet<>(), addPartitionDesc), - context.hiveConf + context.hiveConf, + true ); Task<?> movePartitionTask = movePartitionTask(table, partSpec, tmpPath); @@ -246,7 +247,7 @@ public class LoadPartitions { ); loadTableWork.setInheritTableSpecs(false); MoveWork work = new MoveWork(new HashSet<>(), new HashSet<>(), loadTableWork, null, false); - return TaskFactory.get(work, context.hiveConf); + return TaskFactory.get(work, context.hiveConf, true); } private Path locationOnReplicaWarehouse(Table table, AddPartitionDesc.OnePartitionDesc partSpec) @@ -274,7 +275,8 @@ public class LoadPartitions { desc.getPartition(0).setLocation(ptn.getLocation()); // use existing location return TaskFactory.get( new DDLWork(new HashSet<>(), new HashSet<>(), desc), - context.hiveConf + context.hiveConf, + true ); } http://git-wip-us.apache.org/repos/asf/hive/blob/637224fc/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java index 3b8cb68..e0721f1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java @@ -234,7 +234,7 @@ public class LoadTable { ); MoveWork moveWork = new MoveWork(new HashSet<>(), new HashSet<>(), loadTableWork, null, false); - Task<?> loadTableTask = TaskFactory.get(moveWork, context.hiveConf); + Task<?> loadTableTask = TaskFactory.get(moveWork, context.hiveConf, true); copyTask.addDependentTask(loadTableTask); return copyTask; } http://git-wip-us.apache.org/repos/asf/hive/blob/637224fc/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java index 1d89661..c38f7dc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java @@ -469,7 +469,7 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { alterTblDesc.setPartSpec((HashMap<String, String>)partSpec); Task<? extends Serializable> updateReplIdTask = TaskFactory.get( - new DDLWork(inputs, outputs, alterTblDesc), conf); + new DDLWork(inputs, outputs, alterTblDesc), conf, true); // Link the update repl state task with dependency collection task if (preCursor != null) { @@ -491,7 +491,7 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { AlterDatabaseDesc alterDbDesc = new AlterDatabaseDesc( dbName, mapProp, new ReplicationSpec(replState, replState)); Task<? extends Serializable> updateReplIdTask = TaskFactory.get( - new DDLWork(inputs, outputs, alterDbDesc), conf); + new DDLWork(inputs, outputs, alterDbDesc), conf, true); // Link the update repl state task with dependency collection task if (preCursor != null) { http://git-wip-us.apache.org/repos/asf/hive/blob/637224fc/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java index bdfb632..aef83b8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java @@ -317,10 +317,10 @@ public class ImportTableDesc { public Task<? extends Serializable> getCreateTableTask(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs, HiveConf conf) { switch (getDescType()) { - case TABLE: - return TaskFactory.get(new DDLWork(inputs, outputs, createTblDesc), conf); - case VIEW: - return TaskFactory.get(new DDLWork(inputs, outputs, createViewDesc), conf); + case TABLE: + return TaskFactory.get(new DDLWork(inputs, outputs, createTblDesc), conf, true); + case VIEW: + return TaskFactory.get(new DDLWork(inputs, outputs, createViewDesc), conf, true); } return null; }
