Repository: hive Updated Branches: refs/heads/master 16bfb9c94 -> 90a92b746
http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 0b63a20..fdb2866 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -868,12 +868,13 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { } @Override - public void createTableWithConstraints(Table tbl, + public List<String> createTableWithConstraints(Table tbl, List<SQLPrimaryKey> primaryKeys, List<SQLForeignKey> foreignKeys, List<SQLUniqueConstraint> uniqueConstraints, List<SQLNotNullConstraint> notNullConstraints) throws InvalidObjectException, MetaException { // TODO Auto-generated method stub + return null; } @Override @@ -883,27 +884,29 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { } @Override - public void addPrimaryKeys(List<SQLPrimaryKey> pks) + public List<String> addPrimaryKeys(List<SQLPrimaryKey> pks) throws InvalidObjectException, MetaException { - // TODO Auto-generated method stub + return null; } @Override - public void addForeignKeys(List<SQLForeignKey> fks) + public List<String> addForeignKeys(List<SQLForeignKey> fks) throws InvalidObjectException, MetaException { - // TODO Auto-generated method stub + return null; } @Override - public void addUniqueConstraints(List<SQLUniqueConstraint> uks) + public List<String> addUniqueConstraints(List<SQLUniqueConstraint> uks) throws InvalidObjectException, MetaException { // TODO Auto-generated method stub + return null; } @Override - public void addNotNullConstraints(List<SQLNotNullConstraint> nns) + public List<String> addNotNullConstraints(List<SQLNotNullConstraint> nns) throws InvalidObjectException, MetaException { // TODO Auto-generated method stub + return null; } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 5348011..f422c4e 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -884,12 +884,13 @@ public class DummyRawStoreForJdoConnection implements RawStore { } @Override - public void createTableWithConstraints(Table tbl, + public List<String> createTableWithConstraints(Table tbl, List<SQLPrimaryKey> primaryKeys, List<SQLForeignKey> foreignKeys, List<SQLUniqueConstraint> uniqueConstraints, List<SQLNotNullConstraint> notNullConstraints) throws InvalidObjectException, MetaException { // TODO Auto-generated method stub + return null; } @Override @@ -899,26 +900,30 @@ public class DummyRawStoreForJdoConnection implements RawStore { } @Override - public void addPrimaryKeys(List<SQLPrimaryKey> pks) + public List<String> addPrimaryKeys(List<SQLPrimaryKey> pks) throws InvalidObjectException, MetaException { // TODO Auto-generated method stub + return null; } @Override - public void addForeignKeys(List<SQLForeignKey> fks) + public List<String> addForeignKeys(List<SQLForeignKey> fks) throws InvalidObjectException, MetaException { // TODO Auto-generated method stub + return null; } - public void addUniqueConstraints(List<SQLUniqueConstraint> uks) + public List<String> addUniqueConstraints(List<SQLUniqueConstraint> uks) throws InvalidObjectException, MetaException { // TODO Auto-generated method stub + return null; } @Override - public void addNotNullConstraints(List<SQLNotNullConstraint> nns) + public List<String> addNotNullConstraints(List<SQLNotNullConstraint> nns) throws InvalidObjectException, MetaException { // TODO Auto-generated method stub + return null; } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 16c440f..185ac1d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -385,6 +385,12 @@ public class DDLTask extends Task<DDLWork> implements Serializable { AlterTableDesc alterTbl = work.getAlterTblDesc(); if (alterTbl != null) { + if (!allowOperationInReplicationScope(db, alterTbl.getOldName(), null, alterTbl.getReplicationSpec())) { + // no alter, the table is missing either due to drop/rename which follows the alter. + // or the existing table is newer than our update. + LOG.debug("DDLTask: Alter Table is skipped as table {} is newer than update", alterTbl.getOldName()); + return 0; + } if (alterTbl.getOp() == AlterTableTypes.DROPCONSTRAINT ) { return dropConstraint(db, alterTbl); } else if (alterTbl.getOp() == AlterTableTypes.ADDCONSTRAINT) { @@ -3571,13 +3577,6 @@ public class DDLTask extends Task<DDLWork> implements Serializable { * Throws this exception if an unexpected error occurs. */ private int alterTable(Hive db, AlterTableDesc alterTbl) throws HiveException { - if (!allowOperationInReplicationScope(db, alterTbl.getOldName(), null, alterTbl.getReplicationSpec())) { - // no alter, the table is missing either due to drop/rename which follows the alter. - // or the existing table is newer than our update. - LOG.debug("DDLTask: Alter Table is skipped as table {} is newer than update", alterTbl.getOldName()); - return 0; - } - // alter the table Table tbl = db.getTable(alterTbl.getOldName()); http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 5b7fc25..d595de4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -1782,7 +1782,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { private void analyzeAlterTableDropConstraint(ASTNode ast, String tableName) throws SemanticException { String dropConstraintName = unescapeIdentifier(ast.getChild(0).getText()); - AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, dropConstraintName); + AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, dropConstraintName, (ReplicationSpec)null); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTblDesc), conf)); @@ -1815,7 +1815,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { child.getToken().getText())); } AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, primaryKeys, foreignKeys, - uniqueConstraints); + uniqueConstraints, null); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTblDesc), conf)); http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/DumpType.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/DumpType.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/DumpType.java index 0580546..e982603 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/DumpType.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/DumpType.java @@ -17,8 +17,13 @@ */ package org.apache.hadoop.hive.ql.parse.repl; +import org.apache.hadoop.hive.ql.parse.repl.load.message.AddNotNullConstraintHandler; +import org.apache.hadoop.hive.ql.parse.repl.load.message.AddForeignKeyHandler; +import org.apache.hadoop.hive.ql.parse.repl.load.message.AddPrimaryKeyHandler; +import org.apache.hadoop.hive.ql.parse.repl.load.message.AddUniqueConstraintHandler; import org.apache.hadoop.hive.ql.parse.repl.load.message.CreateFunctionHandler; import org.apache.hadoop.hive.ql.parse.repl.load.message.DefaultHandler; +import org.apache.hadoop.hive.ql.parse.repl.load.message.DropConstraintHandler; import org.apache.hadoop.hive.ql.parse.repl.load.message.DropFunctionHandler; import org.apache.hadoop.hive.ql.parse.repl.load.message.DropPartitionHandler; import org.apache.hadoop.hive.ql.parse.repl.load.message.DropTableHandler; @@ -104,6 +109,36 @@ public enum DumpType { return new InsertHandler(); } }, + EVENT_ADD_PRIMARYKEY("EVENT_ADD_PRIMARYKEY") { + @Override + public MessageHandler handler() { + return new AddPrimaryKeyHandler(); + } + }, + EVENT_ADD_FOREIGNKEY("EVENT_ADD_FOREIGNKEY") { + @Override + public MessageHandler handler() { + return new AddForeignKeyHandler(); + } + }, + EVENT_ADD_UNIQUECONSTRAINT("EVENT_ADD_UNIQUECONSTRAINT") { + @Override + public MessageHandler handler() { + return new AddUniqueConstraintHandler(); + } + }, + EVENT_ADD_NOTNULLCONSTRAINT("EVENT_ADD_NOTNULLCONSTRAINT") { + @Override + public MessageHandler handler() { + return new AddNotNullConstraintHandler(); + } + }, + EVENT_DROP_CONSTRAINT("EVENT_DROP_CONSTRAINT") { + @Override + public MessageHandler handler() { + return new DropConstraintHandler(); + } + }, EVENT_CREATE_FUNCTION("EVENT_CREATE_FUNCTION") { @Override public MessageHandler handler() { http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java new file mode 100644 index 0000000..ccd95fe --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.dump.events; + +import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.ql.parse.repl.DumpType; +import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; + +public class AddForeignKeyHandler extends AbstractEventHandler { + AddForeignKeyHandler(NotificationEvent event) { + super(event); + } + + @Override + public void handle(Context withinContext) throws Exception { + LOG.info("Processing#{} ADD_FOREIGNKEY_MESSAGE message : {}", fromEventId(), event.getMessage()); + DumpMetaData dmd = withinContext.createDmd(this); + dmd.setPayload(event.getMessage()); + dmd.write(); + } + + @Override + public DumpType dumpType() { + return DumpType.EVENT_ADD_FOREIGNKEY; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java new file mode 100644 index 0000000..7973eb3 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.dump.events; + +import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.ql.parse.repl.DumpType; +import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; + +public class AddNotNullConstraintHandler extends AbstractEventHandler { + AddNotNullConstraintHandler(NotificationEvent event) { + super(event); + } + + @Override + public void handle(Context withinContext) throws Exception { + LOG.info("Processing#{} ADD_NOTNULLCONSTRAINT_MESSAGE message : {}", fromEventId(), event.getMessage()); + DumpMetaData dmd = withinContext.createDmd(this); + dmd.setPayload(event.getMessage()); + dmd.write(); + } + + @Override + public DumpType dumpType() { + return DumpType.EVENT_ADD_NOTNULLCONSTRAINT; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java new file mode 100644 index 0000000..3fb3e8b --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.dump.events; + +import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.ql.parse.repl.DumpType; +import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; + +public class AddPrimaryKeyHandler extends AbstractEventHandler { + AddPrimaryKeyHandler(NotificationEvent event) { + super(event); + } + + @Override + public void handle(Context withinContext) throws Exception { + LOG.info("Processing#{} ADD_PRIMARYKEY_MESSAGE message : {}", fromEventId(), event.getMessage()); + DumpMetaData dmd = withinContext.createDmd(this); + dmd.setPayload(event.getMessage()); + dmd.write(); + } + + @Override + public DumpType dumpType() { + return DumpType.EVENT_ADD_PRIMARYKEY; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java new file mode 100644 index 0000000..e4f5415 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.dump.events; + +import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.ql.parse.repl.DumpType; +import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; + +public class AddUniqueConstraintHandler extends AbstractEventHandler { + AddUniqueConstraintHandler(NotificationEvent event) { + super(event); + } + + @Override + public void handle(Context withinContext) throws Exception { + LOG.info("Processing#{} ADD_UNIQUECONSTRAINT_MESSAGE message : {}", fromEventId(), event.getMessage()); + DumpMetaData dmd = withinContext.createDmd(this); + dmd.setPayload(event.getMessage()); + dmd.write(); + } + + @Override + public DumpType dumpType() { + return DumpType.EVENT_ADD_UNIQUECONSTRAINT; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/DropConstraintHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/DropConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/DropConstraintHandler.java new file mode 100644 index 0000000..6b709a6 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/DropConstraintHandler.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.dump.events; + +import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.ql.parse.repl.DumpType; +import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; + +public class DropConstraintHandler extends AbstractEventHandler { + DropConstraintHandler(NotificationEvent event) { + super(event); + } + + @Override + public void handle(Context withinContext) throws Exception { + LOG.info("Processing#{} DROP_CONSTRAINT_MESSAGE message : {}", fromEventId(), event.getMessage()); + DumpMetaData dmd = withinContext.createDmd(this); + dmd.setPayload(event.getMessage()); + dmd.write(); + } + + @Override + public DumpType dumpType() { + return DumpType.EVENT_DROP_CONSTRAINT; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/EventHandlerFactory.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/EventHandlerFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/EventHandlerFactory.java index 7e655fa..4a68235 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/EventHandlerFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/EventHandlerFactory.java @@ -42,6 +42,11 @@ public class EventHandlerFactory { register(MessageFactory.DROP_TABLE_EVENT, DropTableHandler.class); register(MessageFactory.INSERT_EVENT, InsertHandler.class); register(MessageFactory.DROP_FUNCTION_EVENT, DropFunctionHandler.class); + register(MessageFactory.ADD_PRIMARYKEY_EVENT, AddPrimaryKeyHandler.class); + register(MessageFactory.ADD_FOREIGNKEY_EVENT, AddForeignKeyHandler.class); + register(MessageFactory.ADD_UNIQUECONSTRAINT_EVENT, AddUniqueConstraintHandler.class); + register(MessageFactory.ADD_NOTNULLCONSTRAINT_EVENT, AddNotNullConstraintHandler.class); + register(MessageFactory.DROP_CONSTRAINT_EVENT, DropConstraintHandler.class); } static void register(String event, Class<? extends EventHandler> handlerClazz) { http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java index f12bd13..24906a9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java @@ -66,9 +66,11 @@ public class DumpMetaData { } private void loadDumpFromFile() throws SemanticException { - try (FileSystem fs = dumpFile.getFileSystem(hiveConf); BufferedReader br = - new BufferedReader(new InputStreamReader(fs.open(dumpFile)))) { + BufferedReader br = null; + try { // read from dumpfile and instantiate self + FileSystem fs = dumpFile.getFileSystem(hiveConf); + br = new BufferedReader(new InputStreamReader(fs.open(dumpFile))); String line = null; if ((line = br.readLine()) != null) { String[] lineContents = line.split("\t", 5); @@ -83,6 +85,14 @@ public class DumpMetaData { } } catch (IOException ioe) { throw new SemanticException(ioe); + } finally { + if (br != null) { + try { + br.close(); + } catch (IOException e) { + throw new SemanticException(e); + } + } } } http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java new file mode 100644 index 0000000..39697bb --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.load.message; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.hadoop.hive.metastore.api.SQLForeignKey; +import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; +import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; +import org.apache.hadoop.hive.metastore.messaging.AddForeignKeyMessage; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.AlterTableDesc; +import org.apache.hadoop.hive.ql.plan.DDLWork; + +public class AddForeignKeyHandler extends AbstractMessageHandler { + @Override + public List<Task<? extends Serializable>> handle(Context context) + throws SemanticException { + AddForeignKeyMessage msg = deserializer.getAddForeignKeyMessage(context.dmd.getPayload()); + + List<SQLForeignKey> fks = null; + try { + fks = msg.getForeignKeys(); + } catch (Exception e) { + if (!(e instanceof SemanticException)){ + throw new SemanticException("Error reading message members", e); + } else { + throw (SemanticException)e; + } + } + + String actualDbName = context.isDbNameEmpty() ? fks.get(0).getFktable_db() : context.dbName; + String actualTblName = context.isTableNameEmpty() ? fks.get(0).getPktable_name() : context.tableName; + + for (SQLForeignKey fk : fks) { + fk.setPktable_db(actualDbName); + fk.setPktable_name(actualTblName); + fk.setFktable_db(actualDbName); + } + + AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, new ArrayList<SQLPrimaryKey>(), fks, + new ArrayList<SQLUniqueConstraint>(), context.eventOnlyReplicationSpec()); + Task<DDLWork> addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf); + List<Task<? extends Serializable>> tasks = new ArrayList<Task<? extends Serializable>>(); + tasks.add(addConstraintsTask); + context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName); + databasesUpdated.put(actualDbName, context.dmd.getEventTo()); + tablesUpdated.put(actualDbName + "." + actualTblName, context.dmd.getEventTo()); + return Collections.singletonList(addConstraintsTask); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java new file mode 100644 index 0000000..e2c1d1d --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.load.message; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.hadoop.hive.metastore.api.SQLForeignKey; +import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; +import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; +import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; +import org.apache.hadoop.hive.metastore.messaging.AddNotNullConstraintMessage; +import org.apache.hadoop.hive.metastore.messaging.AddUniqueConstraintMessage; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.AlterTableDesc; +import org.apache.hadoop.hive.ql.plan.DDLWork; + +public class AddNotNullConstraintHandler extends AbstractMessageHandler { + @Override + public List<Task<? extends Serializable>> handle(Context context) + throws SemanticException { + AddNotNullConstraintMessage msg = deserializer.getAddNotNullConstraintMessage(context.dmd.getPayload()); + + List<SQLNotNullConstraint> nns = null; + try { + nns = msg.getNotNullConstraints(); + } catch (Exception e) { + if (!(e instanceof SemanticException)){ + throw new SemanticException("Error reading message members", e); + } else { + throw (SemanticException)e; + } + } + + String actualDbName = context.isDbNameEmpty() ? nns.get(0).getTable_db() : context.dbName; + String actualTblName = context.isTableNameEmpty() ? nns.get(0).getTable_name() : context.tableName; + + for (SQLNotNullConstraint nn : nns) { + nn.setTable_db(actualDbName); + nn.setTable_name(actualTblName); + } + + AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, new ArrayList<SQLPrimaryKey>(), new ArrayList<SQLForeignKey>(), + new ArrayList<SQLUniqueConstraint>(), nns, context.eventOnlyReplicationSpec()); + Task<DDLWork> addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf); + List<Task<? extends Serializable>> tasks = new ArrayList<Task<? extends Serializable>>(); + tasks.add(addConstraintsTask); + context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName); + databasesUpdated.put(actualDbName, context.dmd.getEventTo()); + tablesUpdated.put(actualDbName + "." + actualTblName, context.dmd.getEventTo()); + return Collections.singletonList(addConstraintsTask); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java new file mode 100644 index 0000000..7babb6a --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.load.message; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.hadoop.hive.metastore.api.SQLForeignKey; +import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; +import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; +import org.apache.hadoop.hive.metastore.messaging.AddPrimaryKeyMessage; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.AlterTableDesc; +import org.apache.hadoop.hive.ql.plan.DDLWork; + +public class AddPrimaryKeyHandler extends AbstractMessageHandler { + @Override + public List<Task<? extends Serializable>> handle(Context context) + throws SemanticException { + AddPrimaryKeyMessage msg = deserializer.getAddPrimaryKeyMessage(context.dmd.getPayload()); + + List<SQLPrimaryKey> pks = null; + try { + pks = msg.getPrimaryKeys(); + } catch (Exception e) { + if (!(e instanceof SemanticException)){ + throw new SemanticException("Error reading message members", e); + } else { + throw (SemanticException)e; + } + } + String actualDbName = context.isDbNameEmpty() ? pks.get(0).getTable_db() : context.dbName; + String actualTblName = context.isTableNameEmpty() ? pks.get(0).getTable_name() : context.tableName; + + for (SQLPrimaryKey pk : pks) { + pk.setTable_db(actualDbName); + pk.setTable_name(actualTblName); + } + + AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, pks, new ArrayList<SQLForeignKey>(), + new ArrayList<SQLUniqueConstraint>(), context.eventOnlyReplicationSpec()); + Task<DDLWork> addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf); + List<Task<? extends Serializable>> tasks = new ArrayList<Task<? extends Serializable>>(); + tasks.add(addConstraintsTask); + context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName); + databasesUpdated.put(actualDbName, context.dmd.getEventTo()); + tablesUpdated.put(actualDbName + "." + actualTblName, context.dmd.getEventTo()); + return Collections.singletonList(addConstraintsTask); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java new file mode 100644 index 0000000..e7b404a --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.load.message; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.hadoop.hive.metastore.api.SQLForeignKey; +import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; +import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; +import org.apache.hadoop.hive.metastore.messaging.AddUniqueConstraintMessage; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.AlterTableDesc; +import org.apache.hadoop.hive.ql.plan.DDLWork; + +public class AddUniqueConstraintHandler extends AbstractMessageHandler { + @Override + public List<Task<? extends Serializable>> handle(Context context) + throws SemanticException { + AddUniqueConstraintMessage msg = deserializer.getAddUniqueConstraintMessage(context.dmd.getPayload()); + + List<SQLUniqueConstraint> uks = null; + try { + uks = msg.getUniqueConstraints(); + } catch (Exception e) { + if (!(e instanceof SemanticException)){ + throw new SemanticException("Error reading message members", e); + } else { + throw (SemanticException)e; + } + } + + String actualDbName = context.isDbNameEmpty() ? uks.get(0).getTable_db() : context.dbName; + String actualTblName = context.isTableNameEmpty() ? uks.get(0).getTable_name() : context.tableName; + + for (SQLUniqueConstraint uk : uks) { + uk.setTable_db(actualDbName); + uk.setTable_name(actualTblName); + } + + AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, new ArrayList<SQLPrimaryKey>(), new ArrayList<SQLForeignKey>(), + uks, context.eventOnlyReplicationSpec()); + Task<DDLWork> addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf); + List<Task<? extends Serializable>> tasks = new ArrayList<Task<? extends Serializable>>(); + tasks.add(addConstraintsTask); + context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName); + databasesUpdated.put(actualDbName, context.dmd.getEventTo()); + tablesUpdated.put(actualDbName + "." + actualTblName, context.dmd.getEventTo()); + return Collections.singletonList(addConstraintsTask); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java new file mode 100644 index 0000000..58aa214 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.load.message; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.hadoop.hive.metastore.messaging.DropConstraintMessage; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.AlterTableDesc; +import org.apache.hadoop.hive.ql.plan.DDLWork; + +public class DropConstraintHandler extends AbstractMessageHandler { + @Override + public List<Task<? extends Serializable>> handle(Context context) + throws SemanticException { + DropConstraintMessage msg = deserializer.getDropConstraintMessage(context.dmd.getPayload()); + String actualDbName = context.isDbNameEmpty() ? msg.getDB() : context.dbName; + String actualTblName = context.isTableNameEmpty() ? msg.getTable() : context.tableName; + String constraintName = msg.getConstraint(); + + AlterTableDesc dropConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, constraintName, + context.eventOnlyReplicationSpec()); + Task<DDLWork> dropConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, dropConstraintsDesc), context.hiveConf); + List<Task<? extends Serializable>> tasks = new ArrayList<Task<? extends Serializable>>(); + tasks.add(dropConstraintsTask); + context.log.debug("Added drop constrain task : {}:{}", dropConstraintsTask.getId(), actualTblName); + databasesUpdated.put(actualDbName, context.dmd.getEventTo()); + tablesUpdated.put(actualDbName + "." + actualTblName, context.dmd.getEventTo()); + return Collections.singletonList(dropConstraintsTask); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/90a92b74/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java index 2691faa..65e375f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java @@ -322,18 +322,33 @@ public class AlterTableDesc extends DDLDesc implements Serializable { this.numberBuckets = numBuckets; } - public AlterTableDesc(String tableName, String dropConstraintName) { + public AlterTableDesc(String tableName, String dropConstraintName, ReplicationSpec replicationSpec) { this.oldName = tableName; this.dropConstraintName = dropConstraintName; + this.replicationSpec = replicationSpec; op = AlterTableTypes.DROPCONSTRAINT; } public AlterTableDesc(String tableName, List<SQLPrimaryKey> primaryKeyCols, - List<SQLForeignKey> foreignKeyCols, List<SQLUniqueConstraint> uniqueConstraintCols) { + List<SQLForeignKey> foreignKeyCols, List<SQLUniqueConstraint> uniqueConstraintCols, + ReplicationSpec replicationSpec) { this.oldName = tableName; this.primaryKeyCols = primaryKeyCols; this.foreignKeyCols = foreignKeyCols; this.uniqueConstraintCols = uniqueConstraintCols; + this.replicationSpec = replicationSpec; + op = AlterTableTypes.ADDCONSTRAINT; + } + + public AlterTableDesc(String tableName, List<SQLPrimaryKey> primaryKeyCols, + List<SQLForeignKey> foreignKeyCols, List<SQLUniqueConstraint> uniqueConstraintCols, + List<SQLNotNullConstraint> notNullConstraintCols, ReplicationSpec replicationSpec) { + this.oldName = tableName; + this.primaryKeyCols = primaryKeyCols; + this.foreignKeyCols = foreignKeyCols; + this.uniqueConstraintCols = uniqueConstraintCols; + this.notNullConstraintCols = notNullConstraintCols; + this.replicationSpec = replicationSpec; op = AlterTableTypes.ADDCONSTRAINT; }
