Updated Branches: refs/heads/master 8683295fc -> e04fd1902
ACCUMULO-1481 creates root table and copies files to the directory on master start-up Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e04fd190 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e04fd190 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e04fd190 Branch: refs/heads/master Commit: e04fd1902b23b8bca8e1c30795f45d9c596f8623 Parents: 8683295 Author: Eric Newton <[email protected]> Authored: Mon Oct 28 17:15:23 2013 -0400 Committer: Eric Newton <[email protected]> Committed: Mon Oct 28 17:15:23 2013 -0400 ---------------------------------------------------------------------- .../org/apache/accumulo/core/conf/Property.java | 2 -- .../core/metadata/MetadataLocationObtainer.java | 2 +- .../accumulo/core/metadata/RootTable.java | 1 + .../apache/accumulo/server/master/Master.java | 25 ++++++++++++++++++-- .../server/master/recovery/HadoopLogCloser.java | 2 +- .../master/state/MetaDataTableScanner.java | 6 +++-- .../accumulo/server/tabletserver/Tablet.java | 9 ++++--- .../server/tabletserver/TabletServer.java | 4 ++-- .../tabletserver/log/SortedLogRecovery.java | 9 ++++++- .../apache/accumulo/server/util/Initialize.java | 15 ++++++++---- .../accumulo/server/util/MetadataTableUtil.java | 15 ++++-------- 11 files changed, 61 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/core/src/main/java/org/apache/accumulo/core/conf/Property.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index 23bae10..134a037 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -594,12 +594,10 @@ public enum Property { public static Map<String,String> getCompactionStrategyOptions(AccumuloConfiguration tableConf) { Map<String,String> longNames = tableConf.getAllPropertiesWithPrefix(Property.TABLE_COMPACTION_STRATEGY_PREFIX); - log.info("longNames " + longNames); Map<String,String> result = new HashMap<String, String>(); for (Entry<String,String> entry : longNames.entrySet()) { result.put(entry.getKey().substring(Property.TABLE_COMPACTION_STRATEGY_PREFIX.getKey().length()), entry.getValue()); } - log.info("result " + result); return result; } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java index 7d6312d..0fb1a8e 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java @@ -113,7 +113,7 @@ public class MetadataLocationObtainer implements TabletLocationObtainer { if (opTimer != null) opTimer.stop("Got " + results.size() + " results from " + src.tablet_extent + " in %DURATION%"); - // System.out.println("results "+results.keySet()); + //if (log.isTraceEnabled()) log.trace("results "+results); return MetadataLocationObtainer.getMetadataLocationEntries(results); http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java b/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java index 402c1cc..a47f183 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java @@ -42,5 +42,6 @@ public class RootTable { public static final String ZROOT_TABLET_WALOGS = ZROOT_TABLET + "/walogs"; public static final KeyExtent EXTENT = new KeyExtent(new Text(ID), null, null); + public static final KeyExtent OLD_EXTENT = new KeyExtent(new Text(MetadataTable.ID), KeyExtent.getMetadataEntry(new Text(MetadataTable.ID), null), null); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/master/Master.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/accumulo/server/master/Master.java b/server/src/main/java/org/apache/accumulo/server/master/Master.java index 9cbfb51..2c19b6f 100644 --- a/server/src/main/java/org/apache/accumulo/server/master/Master.java +++ b/server/src/main/java/org/apache/accumulo/server/master/Master.java @@ -78,6 +78,7 @@ import org.apache.accumulo.core.security.Credentials; import org.apache.accumulo.core.security.SecurityUtil; import org.apache.accumulo.core.security.thrift.TCredentials; import org.apache.accumulo.core.util.ByteBufferUtil; +import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.Daemon; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.zookeeper.ZooUtil; @@ -95,6 +96,7 @@ import org.apache.accumulo.server.ServerOpts; import org.apache.accumulo.server.client.HdfsZooInstance; import org.apache.accumulo.server.conf.ServerConfiguration; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeManager.FileType; import org.apache.accumulo.server.fs.VolumeManagerImpl; import org.apache.accumulo.server.master.LiveTServerSet.TServerConnection; import org.apache.accumulo.server.master.balancer.DefaultLoadBalancer; @@ -135,6 +137,7 @@ import org.apache.accumulo.server.security.SecurityOperation; import org.apache.accumulo.server.security.SystemCredentials; import org.apache.accumulo.server.util.DefaultMap; import org.apache.accumulo.server.util.Halt; +import org.apache.accumulo.server.util.Initialize; import org.apache.accumulo.server.util.MetadataTableUtil; import org.apache.accumulo.server.util.SystemPropUtil; import org.apache.accumulo.server.util.TServerUtils; @@ -146,6 +149,9 @@ import org.apache.accumulo.server.zookeeper.ZooLock; import org.apache.accumulo.server.zookeeper.ZooReaderWriter; import org.apache.accumulo.trace.instrument.thrift.TraceWrap; import org.apache.accumulo.trace.thrift.TInfo; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; @@ -265,9 +271,13 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt IZooReaderWriter zoo = ZooReaderWriter.getInstance(); + if (!Tables.exists(instance, RootTable.ID)) { + TableManager.prepareNewTableState(instance.getInstanceID(), RootTable.ID, RootTable.NAME, TableState.ONLINE, NodeExistsPolicy.FAIL); + Initialize.initMetadataConfig(RootTable.ID); + } + zoo.recursiveDelete(ZooUtil.getRoot(instance) + "/loggers", NodeMissingPolicy.SKIP); zoo.recursiveDelete(ZooUtil.getRoot(instance) + "/dead/loggers", NodeMissingPolicy.SKIP); - zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZRECOVERY, new byte[] {'0'}, NodeExistsPolicy.SKIP); for (String id : Tables.getIdToNameMap(instance).keySet()) { @@ -293,7 +303,18 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt @Override public void run() { try { - MetadataTableUtil.convertRootTabletToRootTable(instance, SystemCredentials.get()); + Path oldDir = fs.getFullPath(FileType.TABLE, ServerConstants.getDefaultBaseDir() + "/tables/!0/root_tablet"); + for (FileStatus file : fs.listStatus(oldDir)) { + if (file.isFile()) { + Path newFile = new Path(ServerConstants.getRootTabletDir(), file.getPath().getName()); + FileUtil.copy( + fs.getFileSystemByPath(file.getPath()), file.getPath(), + fs.getFileSystemByPath(newFile), newFile, + false, + true, + CachedConfiguration.getInstance()); + } + } MetadataTableUtil.moveMetaDeleteMarkers(instance, SystemCredentials.get()); Accumulo.updateAccumuloVersion(fs); http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java b/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java index 3e7d406..3eda844 100644 --- a/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java +++ b/server/src/main/java/org/apache/accumulo/server/master/recovery/HadoopLogCloser.java @@ -46,7 +46,7 @@ public class HadoopLogCloser implements LogCloser { } catch (FileNotFoundException ex) { throw ex; } catch (Exception ex) { - log.warn("Error recovery lease on " + source.toString(), ex); + log.warn("Error recovering lease on " + source.toString(), ex); ns.append(source).close(); log.info("Recovered lease on " + source.toString() + " using append"); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java b/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java index cec0bcf..1932958 100644 --- a/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java +++ b/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java @@ -37,6 +37,7 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.user.WholeRowIterator; import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily; @@ -49,7 +50,7 @@ import org.apache.log4j.Logger; public class MetaDataTableScanner implements Iterator<TabletLocationState> { private static final Logger log = Logger.getLogger(MetaDataTableScanner.class); - BatchScanner mdScanner; + BatchScanner mdScanner = null; Iterator<Entry<Key,Value>> iter; public MetaDataTableScanner(Instance instance, Credentials credentials, Range range, CurrentState state) { @@ -65,7 +66,8 @@ public class MetaDataTableScanner implements Iterator<TabletLocationState> { mdScanner.setRanges(Collections.singletonList(range)); iter = mdScanner.iterator(); } catch (Exception ex) { - mdScanner.close(); + if (mdScanner != null) + mdScanner.close(); throw new RuntimeException(ex); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java index c734ff7..72c78c3 100644 --- a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java +++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java @@ -1271,7 +1271,6 @@ public class Tablet { this.lastCompactID = initCompactID; if (extent.isRootTablet()) { - long rtime = Long.MIN_VALUE; for (FileRef ref : datafiles.keySet()) { Path path = ref.path(); @@ -1295,6 +1294,10 @@ public class Tablet { } } } + if (time == null && datafiles.isEmpty() && extent.equals(RootTable.OLD_EXTENT)) { + // recovery... old root tablet has no data, so time doesn't matter: + time = TabletTime.LOGICAL_TIME_ID + "" + Long.MIN_VALUE; + } this.tabletServer = tabletServer; this.logId = tabletServer.createLogId(extent); @@ -1430,7 +1433,7 @@ public class Tablet { removeOldTemporaryFiles(); } - log.log(TLevel.TABLET_HIST, extent + " opened "); + log.log(TLevel.TABLET_HIST, extent + " opened"); } private void removeOldTemporaryFiles() { @@ -3176,7 +3179,7 @@ public class Tablet { droppedFiles.addAll(inputFiles); if (plan != null) droppedFiles.addAll(plan.deleteFiles); - propogateDeletes = !(droppedFiles.equals(allFiles.keySet())); + propogateDeletes = droppedFiles.equals(allFiles.keySet()); log.debug("Major compaction plan: " + plan + " propogate deletes : " + propogateDeletes); filesToCompact = new HashMap<FileRef,DataFileValue>(allFiles); filesToCompact.keySet().retainAll(inputFiles); http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java index 92aabf3..c1df5d3 100644 --- a/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java +++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java @@ -2987,7 +2987,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu long now = RelativeTime.currentTimeMillis(); List<String> logSet = new ArrayList<String>(); for (DfsLogger log : logs) - logSet.add(log.toString()); + logSet.add(log.getFileName()); MetadataTableUtil.LogEntry entry = new MetadataTableUtil.LogEntry(); entry.extent = extent; entry.tabletId = id; @@ -3376,7 +3376,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu throw new AccumuloException("Metadata entry does not have directory (" + metadataEntry + ")"); } - if (time == null) { + if (time == null && !extent.equals(RootTable.OLD_EXTENT)) { throw new AccumuloException("Metadata entry does not have time (" + metadataEntry + ")"); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java index 840fad0..2f1a875 100644 --- a/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java +++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecovery.java @@ -29,10 +29,13 @@ import java.util.Set; import org.apache.accumulo.core.data.KeyExtent; import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.server.fs.VolumeManager; import org.apache.accumulo.server.logger.LogFileKey; import org.apache.accumulo.server.logger.LogFileValue; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; /** @@ -150,6 +153,10 @@ public class SortedLogRecovery { throw new RuntimeException("COMPACTION_FINISH (without preceding COMPACTION_START) is not followed by a successful minor compaction."); lastStartToFinish.update(key.tserverSession); } + KeyExtent alternative = extent; + if (extent.isRootTablet()) { + alternative = RootTable.OLD_EXTENT; + } LogFileKey defineKey = null; @@ -159,7 +166,7 @@ public class SortedLogRecovery { // LogReader.printEntry(entry); if (key.event != DEFINE_TABLET) break; - if (key.tablet.equals(extent)) { + if (key.tablet.equals(extent) || key.tablet.equals(alternative)) { if (tid != key.tid) { tid = key.tid; defineKey = key; http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/util/Initialize.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/accumulo/server/util/Initialize.java b/server/src/main/java/org/apache/accumulo/server/util/Initialize.java index f7ff87a..3f6ef19 100644 --- a/server/src/main/java/org/apache/accumulo/server/util/Initialize.java +++ b/server/src/main/java/org/apache/accumulo/server/util/Initialize.java @@ -237,7 +237,7 @@ public class Initialize { for (Path mtd : metadataTableDirs) { try { fstat = fs.getFileStatus(mtd); - if (!fstat.isDir()) { + if (!fstat.isDirectory()) { log.fatal("location " + mtd.toString() + " exists but is not a directory"); return; } @@ -252,7 +252,7 @@ public class Initialize { // create root table and tablet try { fstat = fs.getFileStatus(rootTablet); - if (!fstat.isDir()) { + if (!fstat.isDirectory()) { log.fatal("location " + rootTablet.toString() + " exists but is not a directory"); return; } @@ -313,14 +313,14 @@ public class Initialize { Path dir = new Path(s); try { fstat = fs.getFileStatus(dir); - if (!fstat.isDir()) { + if (!fstat.isDirectory()) { log.fatal("location " + dir.toString() + " exists but is not a directory"); return; } } catch (FileNotFoundException fnfe) { try { fstat = fs.getFileStatus(dir); - if (!fstat.isDir()) { + if (!fstat.isDirectory()) { log.fatal("location " + dir.toString() + " exists but is not a directory"); return; } @@ -432,7 +432,7 @@ public class Initialize { opts.rootpass); } - protected static void initMetadataConfig() throws IOException { + public static void initMetadataConfig(String tableId) throws IOException { try { Configuration conf = CachedConfiguration.getInstance(); int max = conf.getInt("dfs.replication.max", 512); @@ -454,6 +454,11 @@ public class Initialize { } } + protected static void initMetadataConfig() throws IOException { + initMetadataConfig(RootTable.ID); + initMetadataConfig(MetadataTable.ID); + } + private static void setMetadataReplication(int replication, String reason) throws IOException { String rep = getConsoleReader().readLine( "Your HDFS replication " + reason + " is not compatible with our default " + MetadataTable.NAME + " replication of 5. What do you want to set your " http://git-wip-us.apache.org/repos/asf/accumulo/blob/e04fd190/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java index f5ed91b..b52ec9a 100644 --- a/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java +++ b/server/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java @@ -810,7 +810,7 @@ public class MetadataTableUtil { if (extent.isRootTablet()) { getRootLogEntries(result); Path rootDir = new Path(ServerConstants.getRootTabletDir()); - rootDir = rootDir.makeQualified(fs.getDefaultVolume()); + rootDir = fs.getDefaultVolume().makeQualified(rootDir); FileStatus[] files = fs.listStatus(rootDir); for (FileStatus fileStatus : files) { if (fileStatus.getPath().toString().endsWith("_tmp")) { @@ -892,6 +892,8 @@ public class MetadataTableUtil { LogEntry e = new LogEntry(); try { e.fromBytes(zoo.getData(root + "/" + child, null)); + // upgrade from !0;!0<< -> !!R<< + e.extent = RootTable.EXTENT; result.add(e); } catch (KeeperException.NoNodeException ex) { continue; @@ -1275,11 +1277,11 @@ public class MetadataTableUtil { log.info("Moving " + filename + " marker in " + RootTable.NAME); Mutation m = new Mutation(MetadataSchema.DeletesSection.getRowPrefix() + filename); m.put(EMPTY_BYTES, EMPTY_BYTES, EMPTY_BYTES); - update(creds, m, null); + update(creds, m, RootTable.EXTENT); // remove the old entry m = new Mutation(entry.getKey().getRow()); m.putDelete(EMPTY_BYTES, EMPTY_BYTES); - update(creds, m, null); + update(creds, m, RootTable.OLD_EXTENT); } else { break; } @@ -1315,11 +1317,4 @@ public class MetadataTableUtil { return tabletEntries; } - public static void convertRootTabletToRootTable(Instance instance, SystemCredentials systemCredentials) throws KeeperException, InterruptedException { - ZooReaderWriter zoo = ZooReaderWriter.getInstance(); - if (zoo.exists(ZooUtil.getRoot(instance)+"/tables/" + RootTable.ID)) - return; - TableManager.prepareNewTableState(instance.getInstanceID(), RootTable.ID, RootTable.NAME, TableState.ONLINE, NodeExistsPolicy.FAIL); - } - }
