Author: ctubbsii
Date: Fri Jun 7 19:38:14 2013
New Revision: 1490795
URL: http://svn.apache.org/r1490795
Log:
ACCUMULO-1481 Use METADATA_TABLE_NAME constant wherever it wasn't already being
used
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java?rev=1490795&r1=1490794&r2=1490795&view=diff
==============================================================================
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
Fri Jun 7 19:38:14 2013
@@ -105,23 +105,23 @@ public class SimpleGarbageCollector impl
private static final Text EMPTY_TEXT = new Text();
static class Opts extends Help {
- @Parameter(names={"-v", "--verbose"}, description="extra information will
get printed to stdout also")
+ @Parameter(names = {"-v", "--verbose"}, description = "extra information
will get printed to stdout also")
boolean verbose = false;
- @Parameter(names={"-s", "--safemode"}, description="safe mode will not
delete files")
+ @Parameter(names = {"-s", "--safemode"}, description = "safe mode will not
delete files")
boolean safeMode = false;
- @Parameter(names={"-o", "--offline"}, description=
- "offline mode will run once and check data files directly; this is
dangerous if accumulo is running or not shut down properly")
+ @Parameter(names = {"-o", "--offline"},
+ description = "offline mode will run once and check data files
directly; this is dangerous if accumulo is running or not shut down properly")
boolean offline = false;
- @Parameter(names={"-a", "--address"}, description="specify our local
address")
+ @Parameter(names = {"-a", "--address"}, description = "specify our local
address")
String address = null;
}
-
+
// how much of the JVM's available memory should it use gathering candidates
private static final float CANDIDATE_MEMORY_PERCENTAGE = 0.75f;
private boolean candidateMemExceeded;
private static final Logger log =
Logger.getLogger(SimpleGarbageCollector.class);
-
+
private TCredentials credentials;
private long gcStartDelay;
private boolean checkForBulkProcessingFiles;
@@ -181,7 +181,7 @@ public class SimpleGarbageCollector impl
public void useAddress(String address) {
this.address = address;
}
-
+
public void init(FileSystem fs, Instance instance, TCredentials credentials,
boolean noTrash) throws IOException {
this.fs = TraceFileSystem.wrap(fs);
this.credentials = credentials;
@@ -368,7 +368,7 @@ public class SimpleGarbageCollector impl
try {
tabletDirs = fs.listStatus(new Path(ServerConstants.getTablesDir() +
"/" + delTableId));
} catch (FileNotFoundException ex) {
- // ignored
+ // ignored
}
if (tabletDirs == null)
@@ -376,7 +376,7 @@ public class SimpleGarbageCollector impl
if (tabletDirs.length == 0) {
Path p = new Path(ServerConstants.getTablesDir() + "/" + delTableId);
- if (!moveToTrash(p))
+ if (!moveToTrash(p))
fs.delete(p, false);
}
}
@@ -387,6 +387,7 @@ public class SimpleGarbageCollector impl
String path = ZooUtil.getRoot(HdfsZooInstance.getInstance()) +
Constants.ZGC_LOCK;
LockWatcher lockWatcher = new LockWatcher() {
+ @Override
public void lostLock(LockLossReason reason) {
Halt.halt("GC lock in zookeeper lost (reason = " + reason + "),
exiting!");
}
@@ -450,7 +451,7 @@ public class SimpleGarbageCollector impl
}
return candidates;
}
-
+
checkForBulkProcessingFiles = false;
Range range = Constants.METADATA_DELETES_FOR_METADATA_KEYSPACE;
candidates.addAll(getBatch(Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX,
range));
@@ -461,9 +462,9 @@ public class SimpleGarbageCollector impl
candidates.addAll(getBatch(Constants.METADATA_DELETE_FLAG_PREFIX, range));
return candidates;
}
-
+
private Collection<String> getBatch(String prefix, Range range) throws
Exception {
- // want to ensure GC makes progress... if the 1st N deletes are stable and
we keep processing them,
+ // want to ensure GC makes progress... if the 1st N deletes are stable and
we keep processing them,
// then will never inspect deletes after N
if (continueKey != null) {
if (!range.contains(continueKey)) {
@@ -474,7 +475,8 @@ public class SimpleGarbageCollector impl
continueKey = null;
}
- Scanner scanner = instance.getConnector(credentials.getPrincipal(),
CredentialHelper.extractToken(credentials)).createScanner(Constants.METADATA_TABLE_NAME,
Constants.NO_AUTHS);
+ Scanner scanner = instance.getConnector(credentials.getPrincipal(),
CredentialHelper.extractToken(credentials)).createScanner(
+ Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
scanner.setRange(range);
List<String> result = new ArrayList<String>();
// find candidates for deletion; chop off the prefix
@@ -513,7 +515,8 @@ public class SimpleGarbageCollector impl
}
} else {
try {
- scanner = new
IsolatedScanner(instance.getConnector(credentials.getPrincipal(),
CredentialHelper.extractToken(credentials)).createScanner(Constants.METADATA_TABLE_NAME,
Constants.NO_AUTHS));
+ scanner = new
IsolatedScanner(instance.getConnector(credentials.getPrincipal(),
CredentialHelper.extractToken(credentials)).createScanner(
+ Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS));
} catch (AccumuloSecurityException ex) {
throw new AccumuloException(ex);
} catch (TableNotFoundException ex) {
@@ -593,8 +596,9 @@ public class SimpleGarbageCollector impl
}
}
}
-
+
final static String METADATA_TABLE_DIR = "/" + Constants.METADATA_TABLE_ID;
+
private static void putMarkerDeleteMutation(final String delete, final
BatchWriter writer, final BatchWriter rootWriter) throws
MutationsRejectedException {
if (delete.startsWith(METADATA_TABLE_DIR)) {
Mutation m = new Mutation(new
Text(Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX + delete));
@@ -606,7 +610,7 @@ public class SimpleGarbageCollector impl
writer.addMutation(m);
}
}
-
+
/**
* This method attempts to do its best to remove files from the filesystem
that have been confirmed for deletion.
*/
@@ -622,7 +626,7 @@ public class SimpleGarbageCollector impl
writer = c.createBatchWriter(Constants.METADATA_TABLE_NAME, new
BatchWriterConfig());
rootWriter = c.createBatchWriter(Constants.METADATA_TABLE_NAME, new
BatchWriterConfig());
} catch (Exception e) {
- log.error("Unable to create writer to remove file from the !METADATA
table", e);
+ log.error("Unable to create writer to remove file from the " +
Constants.METADATA_TABLE_NAME + " table", e);
}
}
// when deleting a dir and all files in that dir, only need to delete the
dir
@@ -715,7 +719,7 @@ public class SimpleGarbageCollector impl
}
}
-
+
};
deleteThreadPool.execute(deleteTask);
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1490795&r1=1490794&r2=1490795&view=diff
==============================================================================
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
Fri Jun 7 19:38:14 2013
@@ -278,7 +278,7 @@ public class Master implements LiveTServ
zoo.recursiveDelete(ZooUtil.getRoot(instance) + "/loggers",
NodeMissingPolicy.SKIP);
zoo.recursiveDelete(ZooUtil.getRoot(instance) + "/dead/loggers",
NodeMissingPolicy.SKIP);
-
+
zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZRECOVERY,
new byte[] {'0'}, NodeExistsPolicy.SKIP);
for (String id : Tables.getIdToNameMap(instance).keySet()) {
@@ -606,9 +606,9 @@ public class Master implements LiveTServ
throw new ThriftTableOperationException(tableId, null,
TableOperation.FLUSH, TableOperationExceptionType.NOTFOUND, null);
} catch (AccumuloException e) {
- log.debug("Failed to scan !METADATA table to wait for flush " +
tableId, e);
+ log.debug("Failed to scan " + Constants.METADATA_TABLE_NAME + "
table to wait for flush " + tableId, e);
} catch (TabletDeletedException tde) {
- log.debug("Failed to scan !METADATA table to wait for flush " +
tableId, tde);
+ log.debug("Failed to scan " + Constants.METADATA_TABLE_NAME + "
table to wait for flush " + tableId, tde);
} catch (AccumuloSecurityException e) {
log.warn(e.getMessage(), e);
throw new ThriftSecurityException();
@@ -667,7 +667,7 @@ public class Master implements LiveTServ
throw new Exception("Invalid table property.");
}
} catch (KeeperException.NoNodeException e) {
- // race condition... table no longer exists? This call will throw an
exception if the table was deleted:
+ // race condition... table no longer exists? This call will throw an
exception if the table was deleted:
checkTableId(tableName, op);
log.info("Error altering table property", e);
throw new ThriftTableOperationException(tableId, tableName, op,
TableOperationExceptionType.OTHER, "Problem altering table property");
@@ -2099,11 +2099,8 @@ public class Master implements LiveTServ
});
TCredentials systemAuths = SecurityConstants.getSystemCredentials();
- final TabletStateStore stores[] = {
- new ZooTabletStateStore(new ZooStore(zroot)),
- new RootTabletStateStore(instance, systemAuths, this),
- new MetaDataStateStore(instance, systemAuths, this)
- };
+ final TabletStateStore stores[] = {new ZooTabletStateStore(new
ZooStore(zroot)), new RootTabletStateStore(instance, systemAuths, this),
+ new MetaDataStateStore(instance, systemAuths, this)};
watchers.add(new TabletGroupWatcher(stores[2], null));
watchers.add(new TabletGroupWatcher(stores[1], watchers.get(0)));
watchers.add(new TabletGroupWatcher(stores[0], watchers.get(1)));
@@ -2168,7 +2165,7 @@ public class Master implements LiveTServ
if (acquiredLock || failedToAcquireLock) {
Halt.halt("Zoolock in unexpected state AL " + acquiredLock + " " +
failedToAcquireLock, -1);
}
-
+
acquiredLock = true;
notifyAll();
}
@@ -2180,7 +2177,7 @@ public class Master implements LiveTServ
if (acquiredLock) {
Halt.halt("Zoolock in unexpected state FAL " + acquiredLock + " " +
failedToAcquireLock, -1);
}
-
+
failedToAcquireLock = true;
notifyAll();
}
@@ -2193,10 +2190,10 @@ public class Master implements LiveTServ
}
}
}
-
+
private void getMasterLock(final String zMasterLoc) throws KeeperException,
InterruptedException {
log.info("trying to get master lock");
-
+
final String masterClientAddress =
org.apache.accumulo.core.util.AddressUtil.toString(new
InetSocketAddress(hostname, getSystemConfiguration().getPort(
Property.MASTER_CLIENTPORT)));
@@ -2205,7 +2202,7 @@ public class Master implements LiveTServ
MasterLockWatcher masterLockWatcher = new MasterLockWatcher();
masterLock = new ZooLock(zMasterLoc);
masterLock.lockAsync(masterLockWatcher, masterClientAddress.getBytes());
-
+
masterLockWatcher.waitForChange();
if (masterLockWatcher.acquiredLock) {
@@ -2215,12 +2212,12 @@ public class Master implements LiveTServ
if (!masterLockWatcher.failedToAcquireLock) {
throw new IllegalStateException("master lock in unknown state");
}
-
+
masterLock.tryToCancelAsyncLockOrUnlock();
-
+
UtilWaitThread.sleep(TIME_TO_WAIT_BETWEEN_LOCK_CHECKS);
}
-
+
setMasterState(MasterState.HAVE_LOCK);
}
@@ -2288,7 +2285,7 @@ public class Master implements LiveTServ
}
nextEvent.event("There are now %d tablet servers", current.size());
}
-
+
private static void cleanListByHostAndPort(Collection<TServerInstance>
badServers, Set<TServerInstance> deleted, Set<TServerInstance> added) {
Iterator<TServerInstance> badIter = badServers.iterator();
while (badIter.hasNext()) {
@@ -2307,7 +2304,6 @@ public class Master implements LiveTServ
}
}
}
-
@Override
public void stateChanged(String tableId, TableState state) {
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java?rev=1490795&r1=1490794&r2=1490795&view=diff
==============================================================================
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/ChaoticLoadBalancer.java
Fri Jun 7 19:38:14 2013
@@ -25,6 +25,7 @@ import java.util.Random;
import java.util.Set;
import java.util.SortedMap;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.master.thrift.TableInfo;
@@ -42,9 +43,6 @@ import org.apache.thrift.TException;
public class ChaoticLoadBalancer extends TabletBalancer {
Random r = new Random();
- /* (non-Javadoc)
- * @see
org.apache.accumulo.server.master.balancer.TabletBalancer#getAssignments(java.util.SortedMap,
java.util.Map, java.util.Map)
- */
@Override
public void getAssignments(SortedMap<TServerInstance,TabletServerStatus>
current, Map<KeyExtent,TServerInstance> unassigned,
Map<KeyExtent,TServerInstance> assignments) {
@@ -62,9 +60,8 @@ public class ChaoticLoadBalancer extends
toAssign.put(e.getKey(), avg - numTablets);
}
}
-
- for (KeyExtent ke : unassigned.keySet())
- {
+
+ for (KeyExtent ke : unassigned.keySet()) {
int index = r.nextInt(tServerArray.size());
TServerInstance dest = tServerArray.get(index);
assignments.put(ke, dest);
@@ -85,10 +82,10 @@ public class ChaoticLoadBalancer extends
public long balance(SortedMap<TServerInstance,TabletServerStatus> current,
Set<KeyExtent> migrations, List<TabletMigration> migrationsOut) {
Map<TServerInstance,Long> numTablets = new HashMap<TServerInstance,Long>();
List<TServerInstance> underCapacityTServer = new
ArrayList<TServerInstance>();
-
+
if (!migrations.isEmpty())
return 100;
-
+
boolean moveMetadata = r.nextInt(4) == 0;
long totalTablets = 0;
for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
@@ -104,11 +101,9 @@ public class ChaoticLoadBalancer extends
// *1.2 to handle fuzziness, and prevent locking for 'perfect' balancing
scenarios
long avg = (long) Math.ceil(((double) totalTablets) / current.size() *
1.2);
- for (Entry<TServerInstance, TabletServerStatus> e : current.entrySet())
- {
- for (String table : e.getValue().getTableMap().keySet())
- {
- if (!moveMetadata && "!METADATA".equals(table))
+ for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
+ for (String table : e.getValue().getTableMap().keySet()) {
+ if (!moveMetadata && Constants.METADATA_TABLE_NAME.equals(table))
continue;
try {
for (TabletStats ts : getOnlineTabletsForTable(e.getKey(), table)) {
@@ -146,7 +141,6 @@ public class ChaoticLoadBalancer extends
* @see
org.apache.accumulo.server.master.balancer.TabletBalancer#init(org.apache.accumulo.server.conf.ServerConfiguration)
*/
@Override
- public void init(ServerConfiguration conf) {
- }
+ public void init(ServerConfiguration conf) {}
}
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java?rev=1490795&r1=1490794&r2=1490795&view=diff
==============================================================================
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
Fri Jun 7 19:38:14 2013
@@ -21,8 +21,6 @@ import java.util.Map;
import java.util.Map.Entry;
import org.apache.accumulo.core.Constants;
-import org.apache.accumulo.server.cli.ClientOpts;
-import
org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
@@ -32,6 +30,8 @@ import org.apache.accumulo.core.data.Par
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.server.cli.ClientOpts;
+import
org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.Text;
@@ -62,7 +62,7 @@ public class MergeStats {
public MergeInfo getMergeInfo() {
return info;
}
-
+
public void update(KeyExtent ke, TabletState state, boolean chopped, boolean
hasWALs) {
if (ke.isRootTablet())
return;
@@ -186,7 +186,7 @@ public class MergeStats {
}
scanner.setRange(range);
KeyExtent prevExtent = null;
-
+
log.debug("Scanning range " + range);
for (Entry<Key,Value> entry : scanner) {
TabletLocationState tls;
@@ -200,12 +200,12 @@ public class MergeStats {
if (!tls.extent.getTableId().equals(tableId)) {
break;
}
-
+
if (!tls.walogs.isEmpty() &&
verify.getMergeInfo().needsToBeChopped(tls.extent)) {
log.debug("failing consistency: needs to be chopped" + tls.extent);
return false;
}
-
+
if (prevExtent == null) {
// this is the first tablet observed, it must be offline and its prev
row must be less than the start of the merge range
if (tls.extent.getPrevEndRow() != null &&
tls.extent.getPrevEndRow().compareTo(start) > 0) {
@@ -219,21 +219,20 @@ public class MergeStats {
}
} else if (!tls.extent.isPreviousExtent(prevExtent)) {
- log.debug("hole in !METADATA");
+ log.debug("hole in " + Constants.METADATA_TABLE_NAME);
return false;
}
prevExtent = tls.extent;
-
+
verify.update(tls.extent, tls.getState(master.onlineTabletServers()),
tls.chopped, !tls.walogs.isEmpty());
// stop when we've seen the tablet just beyond our range
if (tls.extent.getPrevEndRow() != null && extent.getEndRow() != null &&
tls.extent.getPrevEndRow().compareTo(extent.getEndRow()) > 0) {
break;
}
}
- log.debug("chopped " + chopped + " v.chopped " + verify.chopped +
- " unassigned " + unassigned + " v.unassigned " + verify.unassigned +
- " verify.total " + verify.total);
+ log.debug("chopped " + chopped + " v.chopped " + verify.chopped + "
unassigned " + unassigned + " v.unassigned " + verify.unassigned + "
verify.total "
+ + verify.total);
return chopped == verify.chopped && unassigned == verify.unassigned &&
unassigned == verify.total;
}
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java?rev=1490795&r1=1490794&r2=1490795&view=diff
==============================================================================
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/tableOps/DeleteTable.java
Fri Jun 7 19:38:14 2013
@@ -109,7 +109,7 @@ class CleanUp extends MasterRepo {
break;
}
}
-
+
if (!done)
return 50;
@@ -145,7 +145,7 @@ class CleanUp extends MasterRepo {
} catch (Exception e) {
refCount = -1;
- log.error("Failed to scan !METADATA looking for references to deleted
table " + tableId, e);
+ log.error("Failed to scan " + Constants.METADATA_TABLE_NAME + " looking
for references to deleted table " + tableId, e);
}
// remove metadata table entries
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java?rev=1490795&r1=1490794&r2=1490795&view=diff
==============================================================================
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
Fri Jun 7 19:38:14 2013
@@ -132,7 +132,6 @@ import org.apache.log4j.Logger;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.NoNodeException;
-
/*
* We need to be able to have the master tell a tabletServer to
* close this file, and the tablet server to handle all pending client reads
@@ -163,7 +162,7 @@ public class Tablet {
enum MinorCompactionReason {
USER, SYSTEM, CLOSE
}
-
+
public class CommitSession {
private int seq;
@@ -445,11 +444,10 @@ public class Tablet {
private volatile long numEntries;
private volatile long numEntriesInMemory;
-
// a count of the amount of data read by the iterators
private AtomicLong scannedCount = new AtomicLong(0);
private Rate scannedRate = new Rate(0.2);
-
+
private ConfigurationObserver configObserver;
private TabletServer tabletServer;
@@ -682,7 +680,7 @@ public class Tablet {
if (extent.isRootTablet()) {
throw new IllegalArgumentException("Can not import files to root
tablet");
}
-
+
synchronized (bulkFileImportLock) {
TCredentials auths = SecurityConstants.getSystemCredentials();
Connector conn;
@@ -708,11 +706,11 @@ public class Tablet {
dfv.setTime(bulkTime);
}
}
-
+
synchronized (timeLock) {
if (bulkTime > persistedTime)
persistedTime = bulkTime;
-
+
MetadataTable.updateTabletDataFile(tid, extent, abs2rel(paths),
tabletTime.getMetadataValue(persistedTime), auths, tabletServer.getLock());
}
}
@@ -1201,7 +1199,7 @@ public class Tablet {
SortedMap<Key,Value> datafilesMetadata;
Text rowName = extent.getMetadataEntry();
-
+
ScannerImpl mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(),
SecurityConstants.getSystemCredentials(), Constants.METADATA_TABLE_ID,
Constants.NO_AUTHS);
@@ -1378,6 +1376,7 @@ public class Tablet {
constraintChecker.set(new ConstraintChecker(getTableConfiguration()));
}
+ @Override
public void propertiesChanged() {
reloadConstraints();
@@ -1388,6 +1387,7 @@ public class Tablet {
}
}
+ @Override
public void propertyChanged(String prop) {
if (prop.startsWith(Property.TABLE_CONSTRAINT_PREFIX.getKey()))
reloadConstraints();
@@ -1402,6 +1402,7 @@ public class Tablet {
}
+ @Override
public void sessionExpired() {
log.debug("Session expired, no longer updating per table props...");
}
@@ -1423,6 +1424,7 @@ public class Tablet {
absPaths.add(rel2abs(relPath, extent));
tabletServer.recover(this, logEntries, absPaths, new
MutationReceiver() {
+ @Override
public void receive(Mutation m) {
// LogReader.printMutation(m);
Collection<ColumnUpdate> muts = m.getUpdates();
@@ -1476,7 +1478,7 @@ public class Tablet {
// TODO this could hang, causing other tablets to fail to load -
ACCUMULO-1292
AccumuloVFSClassLoader.getContextManager().getClassLoader(contextName);
}
-
+
// do this last after tablet is completely setup because it
// could cause major compaction to start
datafileManager = new DatafileManager(datafiles);
@@ -1559,6 +1561,7 @@ public class Tablet {
super(new Key(k), Arrays.copyOf(v.get(), v.get().length));
}
+ @Override
public String toString() {
return key.toString() + "=" + getValue();
}
@@ -1699,7 +1702,7 @@ public class Tablet {
ScanDataSource dataSource = new ScanDataSource(authorizations,
this.defaultSecurityLabel, columns, ssiList, ssio, interruptFlag);
LookupResult result = null;
-
+
try {
SortedKeyValueIterator<Key,Value> iter = new
SourceSwitchingIterator(dataSource);
result = lookup(iter, ranges, columns, results, maxResultSize);
@@ -1745,7 +1748,7 @@ public class Tablet {
boolean endOfTabletReached = false;
while (iter.hasTop()) {
- value = (Value) iter.getTopValue();
+ value = iter.getTopValue();
key = iter.getTopKey();
KVEntry kvEntry = new KVEntry(key, value); // copies key and value
@@ -2108,7 +2111,7 @@ public class Tablet {
if (statsIterator != null) {
statsIterator.report();
}
-
+
}
public void interrupt() {
@@ -2193,6 +2196,7 @@ public class Tablet {
this.mincReason = mincReason;
}
+ @Override
public void run() {
minorCompactionWaitingToStart = false;
minorCompactionInProgress = true;
@@ -2303,7 +2307,7 @@ public class Tablet {
// don't bother trying to get flush id if closed... could be closed
after this check but that is ok... just trying to cut down on uneeded log
messages....
return false;
}
-
+
// get the flush id before the new memmap is made available for write
long flushId;
try {
@@ -2329,7 +2333,7 @@ public class Tablet {
mct.run();
return true;
}
-
+
boolean initiateMinorCompaction(long flushId, MinorCompactionReason
mincReason) {
MinorCompactionTask mct = createMinorCompactionTask(flushId, mincReason);
if (mct == null)
@@ -2416,7 +2420,7 @@ public class Tablet {
throw new RuntimeException(e);
}
}
-
+
Pair<Long,List<IteratorSetting>> getCompactionID() throws NoNodeException {
try {
String zTablePath = Constants.ZROOT + "/" +
HdfsZooInstance.getInstance().getInstanceID() + Constants.ZTABLES + "/" +
extent.getTableId()
@@ -2426,7 +2430,7 @@ public class Tablet {
long compactID = Long.parseLong(tokens[0]);
CompactionIterators iters = new CompactionIterators();
-
+
if (tokens.length > 1) {
Hex hex = new Hex();
ByteArrayInputStream bais = new
ByteArrayInputStream(hex.decode(tokens[1].split("=")[1].getBytes()));
@@ -2445,7 +2449,6 @@ public class Tablet {
iters = new CompactionIterators();
}
}
-
return new Pair<Long,List<IteratorSetting>>(compactID,
iters.getIterators());
} catch (InterruptedException e) {
@@ -2523,7 +2526,7 @@ public class Tablet {
constraintChecker.compareAndSet(cc, ncc);
}
}
-
+
public CommitSession prepareMutationsForCommit(TservConstraintEnv cenv,
List<Mutation> mutations) throws TConstraintViolationException {
ConstraintChecker cc = constraintChecker.get();
@@ -2678,7 +2681,7 @@ public class Tablet {
log.error(e.toString());
}
}
-
+
if (!saveState || tabletMemory.getMemTable().getNumEntries() == 0) {
return;
}
@@ -2800,21 +2803,21 @@ public class Tablet {
Pair<List<LogEntry>,SortedMap<String,DataFileValue>> fileLog =
MetadataTable.getFileAndLogEntries(SecurityConstants.getSystemCredentials(),
extent);
if (fileLog.getFirst().size() != 0) {
- String msg = "Closed tablet " + extent + " has walog entries in
!METADATA " + fileLog.getFirst();
+ String msg = "Closed tablet " + extent + " has walog entries in " +
Constants.METADATA_TABLE_NAME + " " + fileLog.getFirst();
log.error(msg);
throw new RuntimeException(msg);
}
if (extent.isRootTablet()) {
if
(!fileLog.getSecond().keySet().equals(datafileManager.getDatafileSizesRel().keySet()))
{
- String msg = "Data file in !METADATA differ from in memory data " +
extent + " " + fileLog.getSecond().keySet() + " "
+ String msg = "Data file in " + Constants.METADATA_TABLE_NAME + "
differ from in memory data " + extent + " " + fileLog.getSecond().keySet() + "
"
+ datafileManager.getDatafileSizesRel().keySet();
log.error(msg);
throw new RuntimeException(msg);
}
} else {
if
(!fileLog.getSecond().equals(datafileManager.getDatafileSizesRel())) {
- String msg = "Data file in !METADATA differ from in memory data " +
extent + " " + fileLog.getSecond() + " "
+ String msg = "Data file in " + Constants.METADATA_TABLE_NAME + "
differ from in memory data " + extent + " " + fileLog.getSecond() + " "
+ datafileManager.getDatafileSizesRel();
log.error(msg);
throw new RuntimeException(msg);
@@ -2858,6 +2861,7 @@ public class Tablet {
this.reason = reason;
}
+ @Override
public void run() {
CompactionStats majCStats = null;
@@ -2878,7 +2882,7 @@ public class Tablet {
if (reason == MajorCompactionReason.NORMAL &&
needsMajorCompaction(reason))
initiateMajorCompaction(reason);
}
-
+
} catch (RuntimeException E) {
failed = true;
} finally {
@@ -2891,7 +2895,7 @@ public class Tablet {
}
}
- // We used to synchronize on the Tablet before fetching this information,
+ // We used to synchronize on the Tablet before fetching this information,
// but this method is called by the compaction queue thread to re-order
the compactions.
// The compaction queue holds a lock during this sort.
// A tablet lock can be held while putting itself on the queue, so we
can't lock the tablet
@@ -2925,7 +2929,7 @@ public class Tablet {
}
majorCompactionQueued.add(reason);
-
+
tabletResources.executeMajorCompaction(getExtent(), new
CompactionRunner(reason));
return false;
@@ -3017,7 +3021,7 @@ public class Tablet {
Key first = pair.getFirst();
Key last = pair.getSecond();
// If first and last are null, it's an empty file. Add it to the
compact set so it goes away.
- if ( (first == null && last == null) ||
!this.extent.contains(first.getRow()) || !this.extent.contains(last.getRow())) {
+ if ((first == null && last == null) ||
!this.extent.contains(first.getRow()) || !this.extent.contains(last.getRow())) {
result.put(file, entry.getValue().getSize());
}
}
@@ -3258,10 +3262,10 @@ public class Tablet {
return majCStats;
}
}
-
+
compactionIterators = compactionId.getSecond();
}
-
+
// need to handle case where only one file is being major compacted
while (filesToCompact.size() > 0) {
@@ -3300,7 +3304,7 @@ public class Tablet {
log.debug("Starting MajC " + extent + " (" + reason + ") " +
datafileManager.abs2rel(datafileManager.string2path(copy.keySet())) + " --> "
+ datafileManager.abs2rel(new Path(compactTmpName)) + " " +
compactionIterators);
-
+
// always propagate deletes, unless last batch
Compactor compactor = new Compactor(conf, fs, copy, null,
compactTmpName, filesToCompact.size() == 0 ? propogateDeletes : true,
acuTableConf, extent,
cenv, compactionIterators, reason);
@@ -3350,7 +3354,7 @@ public class Tablet {
});
for (Iterator<Entry<String,Long>> iterator =
filesToCompact.entrySet().iterator(); iterator.hasNext();) {
- Entry<String,Long> entry = (Entry<String,Long>) iterator.next();
+ Entry<String,Long> entry = iterator.next();
fileHeap.add(new Pair<String,Long>(entry.getKey(), entry.getValue()));
}
@@ -3387,7 +3391,7 @@ public class Tablet {
majorCompactionInProgress = true;
}
-
+
try {
majCStats = _majorCompact(reason);
if (reason == MajorCompactionReason.CHOP) {
@@ -3444,7 +3448,7 @@ public class Tablet {
this.numEntries = numEntries;
}
-
+
public long getNumEntries() {
return numEntries;
}
@@ -3554,8 +3558,8 @@ public class Tablet {
splitPoint = findSplitRow(datafileManager.getFiles());
else {
Text tsp = new Text(sp);
- splitPoint = new SplitRowSpec(FileUtil.estimatePercentageLTE(fs,
tabletServer.getSystemConfiguration(), extent.getPrevEndRow(),
- extent.getEndRow(), datafileManager.getFiles(), tsp), tsp);
+ splitPoint = new SplitRowSpec(FileUtil.estimatePercentageLTE(fs,
tabletServer.getSystemConfiguration(), extent.getPrevEndRow(),
extent.getEndRow(),
+ datafileManager.getFiles(), tsp), tsp);
}
if (splitPoint == null || splitPoint.row == null) {
@@ -3592,7 +3596,7 @@ public class Tablet {
// finishes.... therefore split could propogate load flags for a
finished bulk load... there is a special iterator
// on the !METADATA table to clean up this type of garbage
Map<String,Long> bulkLoadedFiles =
MetadataTable.getBulkFilesLoaded(SecurityConstants.getSystemCredentials(),
extent);
-
+
MetadataTable.splitTablet(high, extent.getPrevEndRow(), splitRatio,
SecurityConstants.getSystemCredentials(), tabletServer.getLock());
MetadataTable.addNewTablet(low, lowDirectory,
tabletServer.getTabletSession(), lowDatafileSizes, bulkLoadedFiles,
SecurityConstants.getSystemCredentials(), time, lastFlushID,
lastCompactID, tabletServer.getLock());
@@ -3636,7 +3640,7 @@ public class Tablet {
public double scanRate() {
return scannedRate.rate();
}
-
+
public long totalQueries() {
return this.queryCount;
}
@@ -3718,7 +3722,7 @@ public class Tablet {
}
return result;
}
-
+
private Set<String> beginClearingUnusedLogs() {
Set<String> doomed = new HashSet<String>();
@@ -3880,8 +3884,8 @@ public class Tablet {
if (updateMetadata) {
try {
- // if multiple threads were allowed to update this outside of a sync
block, then it would be
- // a race condition
+ // if multiple threads were allowed to update this outside of a sync
block, then it would be
+ // a race condition
MetadataTable.updateTabletCompactID(extent, compactionId,
SecurityConstants.getSystemCredentials(), tabletServer.getLock());
} finally {
synchronized (this) {
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1490795&r1=1490794&r2=1490795&view=diff
==============================================================================
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
Fri Jun 7 19:38:14 2013
@@ -1277,7 +1277,8 @@ public class TabletServer extends Abstra
} else if (keyExtent.isRootTablet()) {
throw new IllegalArgumentException("Cannot batch query root tablet
with other tablets " + threadPoolExtent + " " + keyExtent);
} else if (keyExtent.isMeta() && !threadPoolExtent.isMeta()) {
- throw new IllegalArgumentException("Cannot batch query !METADATA and
non !METADATA tablets " + threadPoolExtent + " " + keyExtent);
+ throw new IllegalArgumentException("Cannot batch query " +
Constants.METADATA_TABLE_NAME + " and non " + Constants.METADATA_TABLE_NAME + "
tablets "
+ + threadPoolExtent + " " + keyExtent);
}
}
@@ -2109,7 +2110,7 @@ public class TabletServer extends Abstra
}
}
}
-
+
@Override
public List<ActiveCompaction> getActiveCompactions(TInfo tinfo,
TCredentials credentials) throws ThriftSecurityException, TException {
try {
@@ -2639,8 +2640,8 @@ public class TabletServer extends Abstra
if (address == null) {
return null;
}
- MasterClientService.Client client = ThriftUtil.getClient(new
MasterClientService.Client.Factory(), address,
- Property.GENERAL_RPC_TIMEOUT, getSystemConfiguration());
+ MasterClientService.Client client = ThriftUtil.getClient(new
MasterClientService.Client.Factory(), address, Property.GENERAL_RPC_TIMEOUT,
+ getSystemConfiguration());
// log.info("Listener API to master has been opened");
return client;
} catch (Exception e) {
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java?rev=1490795&r1=1490794&r2=1490795&view=diff
==============================================================================
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/AddFilesWithMissingEntries.java
Fri Jun 7 19:38:14 2013
@@ -21,7 +21,6 @@ import java.util.Map.Entry;
import java.util.Set;
import org.apache.accumulo.core.Constants;
-import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.core.cli.BatchWriterOpts;
import org.apache.accumulo.core.client.MultiTableBatchWriter;
import org.apache.accumulo.core.client.Scanner;
@@ -32,6 +31,7 @@ import org.apache.accumulo.core.data.Par
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -46,17 +46,13 @@ public class AddFilesWithMissingEntries
static final Logger log = Logger.getLogger(AddFilesWithMissingEntries.class);
public static class Opts extends ClientOpts {
- @Parameter(names="-update", description="Make changes to the !METADATA
table to include missing files")
+ @Parameter(names = "-update", description = "Make changes to the " +
Constants.METADATA_TABLE_NAME + " table to include missing files")
boolean update = false;
}
-
/**
- * A utility to add files to the !METADATA table that are not listed in the
root tablet.
- * This is a recovery tool for someone who knows what they are doing. It
might be better to
- * save off files, and recover your instance by re-initializing and
importing the existing files.
- *
- * @param args
+ * A utility to add files to the {@value Constants#METADATA_TABLE_NAME}
table that are not listed in the root tablet. This is a recovery tool for
someone who
+ * knows what they are doing. It might be better to save off files, and
recover your instance by re-initializing and importing the existing files.
*/
public static void main(String[] args) throws Exception {
Opts opts = new Opts();
@@ -108,7 +104,8 @@ public class AddFilesWithMissingEntries
writer.close();
}
- private static int addUnknownFiles(FileSystem fs, String directory,
Set<String> knownFiles, KeyExtent ke, MultiTableBatchWriter writer, boolean
update) throws Exception {
+ private static int addUnknownFiles(FileSystem fs, String directory,
Set<String> knownFiles, KeyExtent ke, MultiTableBatchWriter writer, boolean
update)
+ throws Exception {
int count = 0;
final String tableId = ke.getTableId().toString();
final Text row = ke.getMetadataEntry();
Modified:
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
URL:
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Initialize.java?rev=1490795&r1=1490794&r2=1490795&view=diff
==============================================================================
---
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
(original)
+++
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
Fri Jun 7 19:38:14 2013
@@ -384,7 +384,7 @@ public class Initialize {
if (opts.clearInstanceName) {
exists = false;
break;
- } else if ((boolean) (exists =
ZooReaderWriter.getInstance().exists(instanceNamePath))) {
+ } else if (exists =
ZooReaderWriter.getInstance().exists(instanceNamePath)) {
String decision = getConsoleReader().readLine("Instance name \"" +
instanceName + "\" exists. Delete existing entry from zookeeper? [Y/N] : ");
if (decision == null)
System.exit(0);
@@ -396,7 +396,7 @@ public class Initialize {
} while (exists);
return instanceNamePath;
}
-
+
private static byte[] getRootPassword(Opts opts) throws IOException {
if (opts.cliPassword != null) {
return opts.cliPassword.getBytes();
@@ -404,7 +404,8 @@ public class Initialize {
String rootpass;
String confirmpass;
do {
- rootpass = getConsoleReader().readLine("Enter initial password for " +
DEFAULT_ROOT_USER + " (this may not be applicable for your security setup): ",
'*');
+ rootpass = getConsoleReader()
+ .readLine("Enter initial password for " + DEFAULT_ROOT_USER + "
(this may not be applicable for your security setup): ", '*');
if (rootpass == null)
System.exit(0);
confirmpass = getConsoleReader().readLine("Confirm initial password for
" + DEFAULT_ROOT_USER + ": ", '*');
@@ -441,8 +442,8 @@ public class Initialize {
private static void setMetadataReplication(int replication, String reason)
throws IOException {
String rep = getConsoleReader().readLine(
- "Your HDFS replication " + reason
- + " is not compatible with our default !METADATA replication of 5.
What do you want to set your !METADATA replication to? (" + replication + ") ");
+ "Your HDFS replication " + reason + " is not compatible with our
default " + Constants.METADATA_TABLE_NAME
+ + " replication of 5. What do you want to set your " +
Constants.METADATA_TABLE_NAME + " replication to? (" + replication + ") ");
if (rep == null || rep.length() == 0)
rep = Integer.toString(replication);
else
@@ -473,7 +474,7 @@ public class Initialize {
public static void main(String[] args) {
Opts opts = new Opts();
opts.parseArgs(Initialize.class.getName(), args);
-
+
try {
SecurityUtil.serverLogin();
Configuration conf = CachedConfiguration.getInstance();
