ACCUMULO-802 fixed problems with some TableOps not locking correctly, fixed null pointer from some tables missing their namespace during randomwalk
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/bf7c1db8 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/bf7c1db8 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/bf7c1db8 Branch: refs/heads/master Commit: bf7c1db8ff3ee7d6514295ac3f3ff4b24254b285 Parents: 7d8d688 Author: Sean Hickey <tallirishll...@gmail.com> Authored: Thu Aug 15 10:58:23 2013 -0400 Committer: Christopher Tubbs <ctubb...@apache.org> Committed: Wed Dec 4 18:46:11 2013 -0500 ---------------------------------------------------------------------- .../admin/TableNamespaceOperationsImpl.java | 6 ++++- .../accumulo/core/client/impl/Tables.java | 2 +- .../server/conf/TableParentConfiguration.java | 3 +-- .../accumulo/server/tables/TableManager.java | 2 +- .../java/org/apache/accumulo/master/Master.java | 12 +++++++-- .../accumulo/master/tableOps/CloneTable.java | 10 +++++--- .../master/tableOps/CreateTableNamespace.java | 1 + .../accumulo/master/tableOps/ImportTable.java | 11 +++++--- .../accumulo/master/tableOps/RenameTable.java | 27 ++++++++++---------- 9 files changed, 46 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/bf7c1db8/core/src/main/java/org/apache/accumulo/core/client/admin/TableNamespaceOperationsImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/TableNamespaceOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/client/admin/TableNamespaceOperationsImpl.java index 694ba89..35fa8ae 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableNamespaceOperationsImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableNamespaceOperationsImpl.java @@ -340,7 +340,11 @@ public class TableNamespaceOperationsImpl extends TableNamespaceOperationsHelper throw new TableNamespaceNotEmptyException(namespaceId, namespace, null); } for (String table : TableNamespaces.getTableNames(instance, namespaceId)) { - getTableOperations().delete(table); + try { + getTableOperations().delete(table); + } catch (TableNotFoundException e) { + log.debug("Table (" + table + ") not found while deleting namespace, probably deleted while we were deleting the rest of the tables"); + } } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/bf7c1db8/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java b/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java index 248bc6b..f80f506 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java +++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Tables.java @@ -63,7 +63,7 @@ public class Tables { String namespaceId = new String(nId, Constants.UTF8); if (!namespaceId.equals(Constants.DEFAULT_TABLE_NAMESPACE_ID) && !namespaceId.equals(Constants.SYSTEM_TABLE_NAMESPACE_ID)) { try { - name += TableNamespaces.getNamespaceName(instance, new String(namespaceId)) + "."; + name += TableNamespaces.getNamespaceName(instance, namespaceId) + "."; } catch (TableNamespaceNotFoundException e) { Log.error("Table (" + tableId + ") contains reference to namespace (" + namespaceId + ") that doesn't exist"); continue; http://git-wip-us.apache.org/repos/asf/accumulo/blob/bf7c1db8/server/base/src/main/java/org/apache/accumulo/server/conf/TableParentConfiguration.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/conf/TableParentConfiguration.java b/server/base/src/main/java/org/apache/accumulo/server/conf/TableParentConfiguration.java index 7590d76..f8d8411 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/conf/TableParentConfiguration.java +++ b/server/base/src/main/java/org/apache/accumulo/server/conf/TableParentConfiguration.java @@ -35,7 +35,6 @@ public class TableParentConfiguration extends TableNamespaceConfiguration { @Override protected String getNamespaceId() { - this.namespaceId = Tables.getNamespace(inst, tableId); - return this.namespaceId; + return Tables.getNamespace(inst, tableId); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/bf7c1db8/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java index 09bf92f..6d1cf08 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java @@ -317,7 +317,7 @@ public class TableManager { public void removeNamespaceFromTable(String tableId, String namespaceId) throws KeeperException, InterruptedException { // actually, revert it to the default namespace. String zPath = Constants.ZROOT + "/" + instance.getInstanceID() + Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_NAMESPACE; - ZooReaderWriter.getRetryingInstance().putPersistentData(zPath, Constants.DEFAULT_TABLE_NAMESPACE.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE); + ZooReaderWriter.getRetryingInstance().putPersistentData(zPath, Constants.DEFAULT_TABLE_NAMESPACE_ID.getBytes(Constants.UTF8), NodeExistsPolicy.OVERWRITE); } public void cloneNamespace(String srcId, String newId, String namespaceName, Map<String,String> propertiesToSet, Set<String> propertiesToExclude, http://git-wip-us.apache.org/repos/asf/accumulo/blob/bf7c1db8/server/master/src/main/java/org/apache/accumulo/master/Master.java ---------------------------------------------------------------------- diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java index aa0d17a..e0c1398 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/Master.java +++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java @@ -946,7 +946,11 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt if (!security.canRenameTable(c, tableId, oldTableName, newTableName)) throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED); - fate.seedTransaction(opid, new TraceRepo<Master>(new RenameTable(tableId, oldTableName, newTableName)), autoCleanup); + try { + fate.seedTransaction(opid, new TraceRepo<Master>(new RenameTable(tableId, oldTableName, newTableName)), autoCleanup); + } catch (TableNamespaceNotFoundException e) { + throw new TException(e.getMessage(), e); + } break; } @@ -1082,7 +1086,11 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt checkNotMetadataTable(tableName, TableOperation.CREATE); checkTableName(tableName, TableOperation.CREATE); - fate.seedTransaction(opid, new TraceRepo<Master>(new ImportTable(c.getPrincipal(), tableName, exportDir)), autoCleanup); + try { + fate.seedTransaction(opid, new TraceRepo<Master>(new ImportTable(c.getPrincipal(), tableName, exportDir)), autoCleanup); + } catch (TableNamespaceNotFoundException e) { + throw new TException(e.getMessage(), e); + } break; } case EXPORT: { http://git-wip-us.apache.org/repos/asf/accumulo/blob/bf7c1db8/server/master/src/main/java/org/apache/accumulo/master/tableOps/CloneTable.java ---------------------------------------------------------------------- diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CloneTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CloneTable.java index f370a3a..97d6234 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CloneTable.java +++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CloneTable.java @@ -21,6 +21,7 @@ import java.util.Map; import java.util.Set; import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.TableNamespaceNotFoundException; import org.apache.accumulo.core.client.impl.TableNamespaces; import org.apache.accumulo.core.client.impl.Tables; import org.apache.accumulo.core.client.impl.thrift.TableOperation; @@ -76,7 +77,7 @@ class FinishCloneTable extends MasterRepo { TableManager.getInstance().transitionTableState(cloneInfo.tableId, TableState.ONLINE); Utils.unreserveTableNamespace(cloneInfo.srcNamespaceId, tid, false); - if (!cloneInfo.namespaceId.equals(cloneInfo.srcNamespaceId)) + if (!cloneInfo.srcNamespaceId.equals(cloneInfo.namespaceId)) Utils.unreserveTableNamespace(cloneInfo.namespaceId, tid, false); Utils.unreserveTable(cloneInfo.srcTableId, tid, false); Utils.unreserveTable(cloneInfo.tableId, tid, true); @@ -132,13 +133,14 @@ class CloneZookeeper extends MasterRepo { private CloneInfo cloneInfo; - public CloneZookeeper(CloneInfo cloneInfo) { + public CloneZookeeper(CloneInfo cloneInfo) throws TableNamespaceNotFoundException { this.cloneInfo = cloneInfo; + Instance inst = HdfsZooInstance.getInstance(); + this.cloneInfo.namespaceId = TableNamespaces.getNamespaceId(inst, Tables.extractNamespace(this.cloneInfo.tableName)); } @Override public long isReady(long tid, Master environment) throws Exception { - cloneInfo.namespaceId = TableNamespaces.getNamespaceId(environment.getInstance(), Tables.extractNamespace(cloneInfo.tableName)); long val = 0; if (!cloneInfo.srcNamespaceId.equals(cloneInfo.namespaceId)) val += Utils.reserveTableNamespace(cloneInfo.namespaceId, tid, false, true, TableOperation.CLONE); @@ -171,7 +173,7 @@ class CloneZookeeper extends MasterRepo { public void undo(long tid, Master environment) throws Exception { Instance instance = HdfsZooInstance.getInstance(); TableManager.getInstance().removeTable(cloneInfo.tableId); - if (!cloneInfo.namespaceId.equals(cloneInfo.srcNamespaceId)) + if (!cloneInfo.srcNamespaceId.equals(cloneInfo.namespaceId)) Utils.unreserveTableNamespace(cloneInfo.namespaceId, tid, false); Utils.unreserveTable(cloneInfo.tableId, tid, true); Tables.clearCache(instance); http://git-wip-us.apache.org/repos/asf/accumulo/blob/bf7c1db8/server/master/src/main/java/org/apache/accumulo/master/tableOps/CreateTableNamespace.java ---------------------------------------------------------------------- diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CreateTableNamespace.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CreateTableNamespace.java index 247ae56..138d77b 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CreateTableNamespace.java +++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CreateTableNamespace.java @@ -124,6 +124,7 @@ class PopulateZookeeperWithNamespace extends MasterRepo { public void undo(long tid, Master master) throws Exception { TableManager.getInstance().removeNamespace(tableNamespaceInfo.namespaceId); Tables.clearCache(master.getInstance()); + Utils.unreserveTableNamespace(tableNamespaceInfo.namespaceId, tid, true); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/bf7c1db8/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java ---------------------------------------------------------------------- diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java index 4405387..c9ed10f 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java +++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java @@ -34,6 +34,7 @@ import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.TableNamespaceNotFoundException; import org.apache.accumulo.core.client.admin.TableOperationsImpl; import org.apache.accumulo.core.client.impl.TableNamespaces; import org.apache.accumulo.core.client.impl.Tables; @@ -419,7 +420,7 @@ class ImportPopulateZookeeper extends MasterRepo { @Override public long isReady(long tid, Master environment) throws Exception { - return Utils.reserveTableNamespace(tableInfo.namespaceId, tid, false, true, TableOperation.IMPORT) + Utils.reserveTable(tableInfo.tableId, tid, true, false, TableOperation.IMPORT); + return Utils.reserveTable(tableInfo.tableId, tid, true, false, TableOperation.IMPORT); } private Map<String,String> getExportedProps(VolumeManager fs) throws Exception { @@ -470,7 +471,6 @@ class ImportPopulateZookeeper extends MasterRepo { public void undo(long tid, Master env) throws Exception { Instance instance = HdfsZooInstance.getInstance(); TableManager.getInstance().removeTable(tableInfo.tableId); - Utils.unreserveTableNamespace(tableInfo.namespaceId, tid, false); Utils.unreserveTable(tableInfo.tableId, tid, true); Tables.clearCache(instance); } @@ -521,16 +521,18 @@ public class ImportTable extends MasterRepo { private ImportedTableInfo tableInfo; - public ImportTable(String user, String tableName, String exportDir) { + public ImportTable(String user, String tableName, String exportDir) throws TableNamespaceNotFoundException { tableInfo = new ImportedTableInfo(); tableInfo.tableName = tableName; tableInfo.user = user; tableInfo.exportDir = exportDir; + Instance inst = HdfsZooInstance.getInstance(); + tableInfo.namespaceId = TableNamespaces.getNamespaceId(inst, Tables.extractNamespace(tableName)); } @Override public long isReady(long tid, Master environment) throws Exception { - return Utils.reserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid); + return Utils.reserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid) + Utils.reserveTableNamespace(tableInfo.namespaceId, tid, false, true, TableOperation.IMPORT); } @Override @@ -610,5 +612,6 @@ public class ImportTable extends MasterRepo { @Override public void undo(long tid, Master env) throws Exception { Utils.unreserveHdfsDirectory(new Path(tableInfo.exportDir).toString(), tid); + Utils.unreserveTableNamespace(tableInfo.namespaceId, tid, false); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/bf7c1db8/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java ---------------------------------------------------------------------- diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java index c34d082..9044247 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java +++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/RenameTable.java @@ -18,6 +18,7 @@ package org.apache.accumulo.master.tableOps; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.TableNamespaceNotFoundException; import org.apache.accumulo.core.client.impl.TableNamespaces; import org.apache.accumulo.core.client.impl.Tables; import org.apache.accumulo.core.client.impl.thrift.TableOperation; @@ -39,20 +40,23 @@ public class RenameTable extends MasterRepo { private String tableId; private String oldTableName; private String newTableName; - private String namespaceId; + private String oldNamespaceId; + private String newNamespaceId; @Override public long isReady(long tid, Master environment) throws Exception { - return Utils.reserveTableNamespace(namespaceId, tid, false, true, TableOperation.RENAME) + return Utils.reserveTableNamespace(oldNamespaceId, tid, false, true, TableOperation.RENAME) + + Utils.reserveTableNamespace(newNamespaceId, tid, false, true, TableOperation.RENAME) + Utils.reserveTable(tableId, tid, true, true, TableOperation.RENAME); } - public RenameTable(String tableId, String oldTableName, String newTableName) { + public RenameTable(String tableId, String oldTableName, String newTableName) throws TableNamespaceNotFoundException { this.tableId = tableId; this.oldTableName = oldTableName; this.newTableName = newTableName; Instance inst = HdfsZooInstance.getInstance(); - this.namespaceId = Tables.getNamespace(inst, tableId); + this.oldNamespaceId = Tables.getNamespace(inst, tableId); + this.newNamespaceId = TableNamespaces.getNamespaceId(inst, Tables.extractNamespace(newTableName)); } @Override @@ -60,14 +64,9 @@ public class RenameTable extends MasterRepo { Instance instance = master.getInstance(); - final String namespace = Tables.extractNamespace(newTableName); - String namespaceId = TableNamespaces.getNamespaceId(instance, namespace); - final String oldNamespace = Tables.extractNamespace(oldTableName); - String oldNamespaceId = TableNamespaces.getNamespaceId(instance, oldNamespace); - - if (!namespaceId.equals(oldNamespaceId)) { + if (!newNamespaceId.equals(oldNamespaceId)) { TableManager tm = TableManager.getInstance(); - tm.addNamespaceToTable(tableId, namespaceId); + tm.addNamespaceToTable(tableId, newNamespaceId); } IZooReaderWriter zoo = ZooReaderWriter.getRetryingInstance(); @@ -98,7 +97,8 @@ public class RenameTable extends MasterRepo { } finally { Utils.tableNameLock.unlock(); Utils.unreserveTable(tableId, tid, true); - Utils.unreserveTableNamespace(this.namespaceId, tid, false); + Utils.unreserveTableNamespace(this.oldNamespaceId, tid, false); + Utils.unreserveTableNamespace(this.newNamespaceId, tid, false); } Logger.getLogger(RenameTable.class).debug("Renamed table " + tableId + " " + oldTableName + " " + newTableName); @@ -108,8 +108,9 @@ public class RenameTable extends MasterRepo { @Override public void undo(long tid, Master env) throws Exception { + Utils.unreserveTableNamespace(newNamespaceId, tid, false); + Utils.unreserveTableNamespace(oldNamespaceId, tid, false); Utils.unreserveTable(tableId, tid, true); - Utils.unreserveTableNamespace(namespaceId, tid, false); } }