Author: ecn
Date: Tue Jun 18 21:20:07 2013
New Revision: 1494310
URL: http://svn.apache.org/r1494310
Log:
ACCUMULO-118 use a configurable class to select volumes for new files
Added:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/RandomVolumeChooser.java
(with props)
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeChooser.java
(with props)
Modified:
accumulo/branches/ACCUMULO-118/core/src/main/java/org/apache/accumulo/core/conf/Property.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
Modified:
accumulo/branches/ACCUMULO-118/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1494310&r1=1494309&r2=1494310&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-118/core/src/main/java/org/apache/accumulo/core/conf/Property.java
(original)
+++
accumulo/branches/ACCUMULO-118/core/src/main/java/org/apache/accumulo/core/conf/Property.java
Tue Jun 18 21:20:07 2013
@@ -101,6 +101,8 @@ public enum Property {
GENERAL_KERBEROS_PRINCIPAL("general.kerberos.principal", "",
PropertyType.STRING, "Name of the kerberos principal to use. _HOST will
automatically be "
+ "replaced by the machines hostname in the hostname portion of the
principal. Leave blank if not using kerberoized hdfs"),
GENERAL_MAX_MESSAGE_SIZE("tserver.server.message.size.max", "1G",
PropertyType.MEMORY, "The maximum size of a message that can be sent to a
tablet server."),
+ GENERAL_VOLUME_CHOOSER("general.volume.chooser",
"org.apache.accumulo.server.fs.RandomVolumeChooser", PropertyType.CLASSNAME,
"The class that will be used to select which volume will be used to create new
files."),
+
// properties that are specific to master server behavior
MASTER_PREFIX("master.", null, PropertyType.PREFIX, "Properties in this
category affect the behavior of the master server"),
Added:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/RandomVolumeChooser.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/RandomVolumeChooser.java?rev=1494310&view=auto
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/RandomVolumeChooser.java
(added)
+++
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/RandomVolumeChooser.java
Tue Jun 18 21:20:07 2013
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.fs;
+
+import java.util.Random;
+
+public class RandomVolumeChooser implements VolumeChooser {
+ Random random = new Random();
+
+ @Override
+ public String choose(String[] options) {
+ return options[random.nextInt(options.length)];
+ }
+
+}
Propchange:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/RandomVolumeChooser.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeChooser.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeChooser.java?rev=1494310&view=auto
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeChooser.java
(added)
+++
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeChooser.java
Tue Jun 18 21:20:07 2013
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.fs;
+
+
+public interface VolumeChooser {
+ String choose(String[] options);
+}
Propchange:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeChooser.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java?rev=1494310&r1=1494309&r2=1494310&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
(original)
+++
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java
Tue Jun 18 21:20:07 2013
@@ -117,4 +117,6 @@ public interface VolumeManager {
// forward to the appropriate FileSystem object
ContentSummary getContentSummary(Path dir) throws IOException;
+
+ String choose(String[] options);
}
Modified:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java?rev=1494310&r1=1494309&r2=1494310&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
(original)
+++
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java
Tue Jun 18 21:20:07 2013
@@ -58,12 +58,18 @@ public class VolumeManagerImpl implement
Map<String, ? extends FileSystem> volumes;
String defaultVolumes;
AccumuloConfiguration conf;
+ VolumeChooser chooser;
protected VolumeManagerImpl(Map<String, ? extends FileSystem> volumes,
String defaultVolume, AccumuloConfiguration conf) {
this.volumes = volumes;
this.defaultVolumes = defaultVolume;
this.conf = conf;
ensureSyncIsEnabled();
+ try {
+
this.getClass().getClassLoader().loadClass(conf.get(Property.GENERAL_VOLUME_CHOOSER));
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
}
public static org.apache.accumulo.server.fs.VolumeManager getLocal() throws
IOException {
@@ -450,4 +456,9 @@ public class VolumeManagerImpl implement
return getFileSystemByPath(dir).getContentSummary(dir);
}
+ @Override
+ public String choose(String[] options) {
+ return chooser.choose(options);
+ }
+
}
Modified:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java?rev=1494310&r1=1494309&r2=1494310&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java
(original)
+++
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/master/recovery/RecoveryManager.java
Tue Jun 18 21:20:07 2013
@@ -23,7 +23,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Random;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -119,10 +118,9 @@ public class RecoveryManager {
log.info("Created zookeeper entry " + path + " with data " + work);
}
- Random random = new Random();
-
public boolean recoverLogs(KeyExtent extent, Collection<Collection<String>>
walogs) throws IOException {
boolean recoveryNeeded = false;
+ ;
for (Collection<String> logs : walogs) {
for (String walog : logs) {
String hostFilename[] = walog.split("/", 2);
@@ -131,10 +129,8 @@ public class RecoveryManager {
String parts[] = filename.split("/");
String sortId = parts[parts.length - 1];
// TODO: ACCUMULO-118: choose recovery directory with extension
- String[] dirs = ServerConstants.getRecoveryDirs();
- String recoveryDir = dirs[random.nextInt(dirs.length)];
- String dest = recoveryDir + "/" + sortId;
- log.debug("Recovering " + filename + " to " + dest + " using sortId "
+ sortId);
+ String dest =
master.getFileSystem().choose(ServerConstants.getRecoveryDirs()) + "/" + sortId;
+ log.debug("Recovering " + filename + " to " + dest);
boolean sortQueued;
synchronized (this) {
Modified:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java?rev=1494310&r1=1494309&r2=1494310&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
(original)
+++
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/tabletserver/log/DfsLogger.java
Tue Jun 18 21:20:07 2013
@@ -31,7 +31,6 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
@@ -232,19 +231,15 @@ public class DfsLogger {
}
}
- // TODO: ACCUMULO-118
- static final Random random = new Random();
-
public synchronized void open(String address) throws IOException {
String filename = UUID.randomUUID().toString();
logger = StringUtil.join(Arrays.asList(address.split(":")), "+");
log.debug("DfsLogger.open() begin");
- String[] wals = ServerConstants.getWalDirs();
+ VolumeManager fs = conf.getFileSystem();
- logPath = new Path(wals[random.nextInt(wals.length)] + "/" + logger + "/"
+ filename);
+ logPath = new Path(fs.choose(ServerConstants.getWalDirs()) + "/" + logger
+ "/" + filename);
try {
- VolumeManager fs = conf.getFileSystem();
short replication = (short)
conf.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION);
if (replication == 0)
replication = fs.getDefaultReplication(logPath);
Modified:
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TabletOperations.java?rev=1494310&r1=1494309&r2=1494310&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
(original)
+++
accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TabletOperations.java
Tue Jun 18 21:20:07 2013
@@ -17,7 +17,6 @@
package org.apache.accumulo.server.util;
import java.io.IOException;
-import java.util.Random;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.util.UtilWaitThread;
@@ -33,27 +32,23 @@ public class TabletOperations {
private static final Logger log = Logger.getLogger(TabletOperations.class);
- private static final Random random = new Random();
-
- // TODO ACCUMULO-118 make the namespace selection pluggable
public static String createTabletDirectory(VolumeManager fs, String tableId,
Text endRow) {
String lowDirectory;
UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
- String[] tablesDirs = ServerConstants.getTablesDirs();
- String randomNamespace = tablesDirs[random.nextInt(tablesDirs.length)];
+ String volume = fs.choose(ServerConstants.getTablesDirs());
while (true) {
try {
if (endRow == null) {
lowDirectory = Constants.DEFAULT_TABLET_LOCATION;
- Path lowDirectoryPath = new Path(randomNamespace + "/" + tableId +
"/" + lowDirectory);
+ Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" +
lowDirectory);
if (fs.exists(lowDirectoryPath) || fs.mkdirs(lowDirectoryPath))
return
lowDirectoryPath.makeQualified(fs.getFileSystemByPath(lowDirectoryPath)).toString();
log.warn("Failed to create " + lowDirectoryPath + " for unknown
reason");
} else {
lowDirectory = "/" + Constants.GENERATED_TABLET_DIRECTORY_PREFIX +
namer.getNextName();
- Path lowDirectoryPath = new Path(randomNamespace + "/" + tableId +
"/" + lowDirectory);
+ Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" +
lowDirectory);
if (fs.exists(lowDirectoryPath))
throw new IllegalStateException("Dir exist when it should not " +
lowDirectoryPath);
if (fs.mkdirs(lowDirectoryPath))
@@ -63,7 +58,7 @@ public class TabletOperations {
log.warn(e);
}
- log.warn("Failed to create dir for tablet in table " + tableId + " in
namespace " + randomNamespace + " + will retry ...");
+ log.warn("Failed to create dir for tablet in table " + tableId + " in
volume " + volume + " + will retry ...");
UtilWaitThread.sleep(3000);
}
