Repository: incubator-blur Updated Branches: refs/heads/master 6092e1172 -> d92fbf629
Adding some more tests and adding default documentation on standard commands. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/d92fbf62 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/d92fbf62 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/d92fbf62 Branch: refs/heads/master Commit: d92fbf62965c8f4d81e15328184b7371f81be3de Parents: 6092e11 Author: Aaron McCurry <[email protected]> Authored: Tue Sep 23 13:57:00 2014 -0400 Committer: Aaron McCurry <[email protected]> Committed: Tue Sep 23 13:57:00 2014 -0400 ---------------------------------------------------------------------- .../org/apache/blur/command/DocumentCount.java | 8 --- .../blur/command/DocumentCountCombiner.java | 12 ---- .../blur/command/DocumentCountNoCombine.java | 11 ---- .../apache/blur/command/BaseCommandManager.java | 58 +++++++++++++++----- .../java/org/apache/blur/command/Command.java | 6 ++ .../blur/command/ControllerCommandManager.java | 2 +- .../blur/command/ShardCommandManager.java | 14 ++++- .../blur/command/ShardCommandManagerTest.java | 38 ++++++++++--- .../org/apache/blur/command/WaitForSeconds.java | 12 +--- .../services/org.apache.blur.command.Commands | 3 +- 10 files changed, 96 insertions(+), 68 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-command/src/main/java/org/apache/blur/command/DocumentCount.java ---------------------------------------------------------------------- diff --git a/blur-command/src/main/java/org/apache/blur/command/DocumentCount.java b/blur-command/src/main/java/org/apache/blur/command/DocumentCount.java index 0eff004..bc104e7 100644 --- a/blur-command/src/main/java/org/apache/blur/command/DocumentCount.java +++ b/blur-command/src/main/java/org/apache/blur/command/DocumentCount.java @@ -18,18 +18,10 @@ package org.apache.blur.command; import java.io.IOException; -import org.apache.blur.command.Command; -import org.apache.blur.command.IndexContext; -import org.apache.blur.command.IndexReadCommand; -import org.apache.blur.command.annotation.Argument; import org.apache.blur.command.annotation.Description; -import org.apache.blur.command.annotation.RequiredArguments; -import org.apache.blur.command.annotation.OptionalArguments; @SuppressWarnings("serial") @Description("Gets the number of visible documents in the index.") -@RequiredArguments({ @Argument(name = "table", value = "The name of the table to execute the document count command.", type = String.class) }) -@OptionalArguments({ @Argument(name = "shard", value = "The shard id to execute the document count command.", type = String.class) }) public class DocumentCount extends Command implements IndexReadCommand<Integer> { private static final String DOC_COUNT = "docCount"; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-command/src/main/java/org/apache/blur/command/DocumentCountCombiner.java ---------------------------------------------------------------------- diff --git a/blur-command/src/main/java/org/apache/blur/command/DocumentCountCombiner.java b/blur-command/src/main/java/org/apache/blur/command/DocumentCountCombiner.java index 8d0abc2..cf64ed8 100644 --- a/blur-command/src/main/java/org/apache/blur/command/DocumentCountCombiner.java +++ b/blur-command/src/main/java/org/apache/blur/command/DocumentCountCombiner.java @@ -20,22 +20,10 @@ import java.io.IOException; import java.util.Map; import java.util.Map.Entry; -import org.apache.blur.command.Command; -import org.apache.blur.command.ClusterCommand; -import org.apache.blur.command.ClusterContext; -import org.apache.blur.command.IndexContext; -import org.apache.blur.command.IndexReadCombiningCommand; -import org.apache.blur.command.Server; -import org.apache.blur.command.Shard; -import org.apache.blur.command.annotation.Argument; import org.apache.blur.command.annotation.Description; -import org.apache.blur.command.annotation.RequiredArguments; -import org.apache.blur.command.annotation.OptionalArguments; @SuppressWarnings("serial") @Description("Gets the number of visible documents in the index.") -@RequiredArguments({ @Argument(name = "table", value = "The name of the table to execute the document count command.", type = String.class) }) -@OptionalArguments({ @Argument(name = "shard", value = "The shard id to execute the document count command.", type = String.class) }) public class DocumentCountCombiner extends Command implements ClusterCommand<Long>, IndexReadCombiningCommand<Integer, Long> { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-command/src/main/java/org/apache/blur/command/DocumentCountNoCombine.java ---------------------------------------------------------------------- diff --git a/blur-command/src/main/java/org/apache/blur/command/DocumentCountNoCombine.java b/blur-command/src/main/java/org/apache/blur/command/DocumentCountNoCombine.java index e7a74c6..3103ff2 100644 --- a/blur-command/src/main/java/org/apache/blur/command/DocumentCountNoCombine.java +++ b/blur-command/src/main/java/org/apache/blur/command/DocumentCountNoCombine.java @@ -20,21 +20,10 @@ import java.io.IOException; import java.util.Map; import java.util.Map.Entry; -import org.apache.blur.command.Command; -import org.apache.blur.command.ClusterCommand; -import org.apache.blur.command.ClusterContext; -import org.apache.blur.command.IndexContext; -import org.apache.blur.command.IndexReadCommand; -import org.apache.blur.command.Shard; -import org.apache.blur.command.annotation.Argument; import org.apache.blur.command.annotation.Description; -import org.apache.blur.command.annotation.RequiredArguments; -import org.apache.blur.command.annotation.OptionalArguments; @SuppressWarnings("serial") @Description("Gets the number of visible documents in the index.") -@RequiredArguments({ @Argument(name = "table", value = "The name of the table to execute the document count command.", type = String.class) }) -@OptionalArguments({ @Argument(name = "shard", value = "The shard id to execute the document count command.", type = String.class) }) public class DocumentCountNoCombine extends Command implements IndexReadCommand<Integer>, ClusterCommand<Long> { private static final String DOC_COUNT_NO_COMBINE = "docCountNoCombine"; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java b/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java index b9c713c..e246b8a 100644 --- a/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java +++ b/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java @@ -126,16 +126,30 @@ public class BaseCommandManager implements Closeable { Class<? extends Command> clazz = command.getClass(); Map<String, String> arguments = new TreeMap<String, String>(); Argument[] args = getArgumentArray(clazz, optional); - if (args == null) { - return arguments; - } - for (Argument argument : args) { - Class<?> type = argument.type(); - arguments.put(argument.name(), ("(" + type.getSimpleName() + ") " + argument.value()).trim()); + addArguments(arguments, args); + if (optional) { + if (!(command instanceof ShardRoute)) { + Argument[] argumentArray = getArgumentArray(Command.class, optional); + addArguments(arguments, argumentArray); + } + } else { + if (!(command instanceof TableRoute)) { + Argument[] argumentArray = getArgumentArray(Command.class, optional); + addArguments(arguments, argumentArray); + } } return arguments; } + private void addArguments(Map<String, String> arguments, Argument[] args) { + if (args != null) { + for (Argument argument : args) { + Class<?> type = argument.type(); + arguments.put(argument.name(), ("(" + type.getSimpleName() + ") " + argument.value()).trim()); + } + } + } + protected Argument[] getArgumentArray(Class<? extends Command> clazz, boolean optional) { if (optional) { OptionalArguments arguments = clazz.getAnnotation(OptionalArguments.class); @@ -165,23 +179,26 @@ public class BaseCommandManager implements Closeable { }; } - public void commandRefresh() throws IOException { - loadNewCommandsFromCommandPath(); + public int commandRefresh() throws IOException { + return loadNewCommandsFromCommandPath(); } - protected synchronized void loadNewCommandsFromCommandPath() throws IOException { + protected synchronized int loadNewCommandsFromCommandPath() throws IOException { Path path = new Path(_commandPath); FileSystem fileSystem = path.getFileSystem(_configuration); FileStatus[] listStatus = fileSystem.listStatus(path); + int changeCount = 0; for (FileStatus fileStatus : listStatus) { BigInteger contentsCheck = checkContents(fileStatus, fileSystem); Path entryPath = fileStatus.getPath(); BigInteger currentValue = _commandPathLastChange.get(entryPath); if (!contentsCheck.equals(currentValue)) { + changeCount++; loadNewCommand(fileSystem, fileStatus, contentsCheck); _commandPathLastChange.put(entryPath, contentsCheck); } } + return changeCount; } protected void loadNewCommand(FileSystem fileSystem, FileStatus fileStatus, BigInteger hashOfContents) @@ -237,6 +254,7 @@ public class BaseCommandManager implements Closeable { protected BigInteger checkContents(FileStatus fileStatus, FileSystem fileSystem) throws IOException { if (fileStatus.isDir()) { + LOG.info("Scanning directory [{0}].", fileStatus.getPath()); BigInteger count = BigInteger.ZERO; Path path = fileStatus.getPath(); FileStatus[] listStatus = fileSystem.listStatus(path); @@ -245,7 +263,15 @@ public class BaseCommandManager implements Closeable { } return count; } else { - return BigInteger.valueOf(fileStatus.getModificationTime()); + int hashCode = fileStatus.getPath().toString().hashCode(); + long modificationTime = fileStatus.getModificationTime(); + long len = fileStatus.getLen(); + BigInteger bi = BigInteger.valueOf(hashCode).add( + BigInteger.valueOf(modificationTime).add(BigInteger.valueOf(len))); + LOG.info("File path hashcode [{0}], mod time [{1}], len [{2}] equals file code [{3}].", + Integer.toString(hashCode), Long.toString(modificationTime), Long.toString(len), + bi.toString(Character.MAX_RADIX)); + return bi; } } @@ -296,7 +322,8 @@ public class BaseCommandManager implements Closeable { } } - protected Response submitDriverCallable(Callable<Response> callable) throws IOException, TimeoutException { + protected Response submitDriverCallable(Callable<Response> callable) throws IOException, TimeoutException, + ExceptionCollector { ExecutionContext executionContext = ExecutionContext.create(); Future<Response> future = _executorServiceDriver.submit(executionContext.wrapCallable(callable)); executionContext.registerDriverFuture(future); @@ -309,7 +336,11 @@ public class BaseCommandManager implements Closeable { } catch (InterruptedException e) { throw new IOException(e); } catch (ExecutionException e) { - throw new IOException(e.getCause()); + Throwable cause = e.getCause(); + if (cause instanceof ExceptionCollector) { + throw (ExceptionCollector) cause; + } + throw new IOException(cause); } catch (java.util.concurrent.TimeoutException e) { LOG.info("Timeout of command [{0}]", executionId); throw new TimeoutException(executionId); @@ -355,7 +386,8 @@ public class BaseCommandManager implements Closeable { return _commandNameLookup.get(clazz); } - protected Map<String, Set<Shard>> getShards(TableContextFactory tableContextFactory, Command command, final Args args, Set<String> tables) throws IOException { + protected Map<String, Set<Shard>> getShards(TableContextFactory tableContextFactory, Command command, + final Args args, Set<String> tables) throws IOException { Map<String, Set<Shard>> shardMap = new TreeMap<String, Set<Shard>>(); if (command instanceof ShardRoute) { ShardRoute shardRoute = (ShardRoute) command; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-core/src/main/java/org/apache/blur/command/Command.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/Command.java b/blur-core/src/main/java/org/apache/blur/command/Command.java index a063629..fadcd28 100644 --- a/blur-core/src/main/java/org/apache/blur/command/Command.java +++ b/blur-core/src/main/java/org/apache/blur/command/Command.java @@ -18,7 +18,13 @@ package org.apache.blur.command; import java.io.Serializable; +import org.apache.blur.command.annotation.Argument; +import org.apache.blur.command.annotation.OptionalArguments; +import org.apache.blur.command.annotation.RequiredArguments; + @SuppressWarnings("serial") +@RequiredArguments({ @Argument(name = "table", value = "The name of the table to execute the document count command.", type = String.class) }) +@OptionalArguments({ @Argument(name = "shard", value = "The shard id to execute the document count command.", type = String.class) }) public abstract class Command implements Serializable, Cloneable { public abstract String getName(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-core/src/main/java/org/apache/blur/command/ControllerCommandManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/ControllerCommandManager.java b/blur-core/src/main/java/org/apache/blur/command/ControllerCommandManager.java index e33b8be..b30e23d 100644 --- a/blur-core/src/main/java/org/apache/blur/command/ControllerCommandManager.java +++ b/blur-core/src/main/java/org/apache/blur/command/ControllerCommandManager.java @@ -33,7 +33,7 @@ public class ControllerCommandManager extends BaseCommandManager { } public Response execute(TableContextFactory tableContextFactory, LayoutFactory layoutFactory, String commandName, - final Args args) throws IOException, TimeoutException { + final Args args) throws IOException, TimeoutException, ExceptionCollector { final ClusterContext context = createCommandContext(tableContextFactory, layoutFactory, args); final Command command = getCommandObject(commandName); if (command == null) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-core/src/main/java/org/apache/blur/command/ShardCommandManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/ShardCommandManager.java b/blur-core/src/main/java/org/apache/blur/command/ShardCommandManager.java index 92909fe..b9565b5 100644 --- a/blur-core/src/main/java/org/apache/blur/command/ShardCommandManager.java +++ b/blur-core/src/main/java/org/apache/blur/command/ShardCommandManager.java @@ -47,7 +47,7 @@ public class ShardCommandManager extends BaseCommandManager { } public Response execute(final TableContextFactory tableContextFactory, final String commandName, final Args args) - throws IOException, TimeoutException { + throws IOException, TimeoutException, ExceptionCollector { final ShardServerContext shardServerContext = getShardServerContext(); Callable<Response> callable = new Callable<Response>() { @Override @@ -106,7 +106,7 @@ public class ShardCommandManager extends BaseCommandManager { } private Map<Shard, Object> executeReadCommand(ShardServerContext shardServerContext, Command command, - final TableContextFactory tableContextFactory, final Args args) throws IOException { + final TableContextFactory tableContextFactory, final Args args) throws IOException, ExceptionCollector { Set<String> tables = getTables(command, args); if (tables.isEmpty()) { throw new IOException("At least one table needs to specified."); @@ -145,6 +145,7 @@ public class ShardCommandManager extends BaseCommandManager { } } Map<Shard, Object> resultMap = new HashMap<Shard, Object>(); + ExceptionCollector collector = null; for (Entry<Shard, Future<?>> e : futureMap.entrySet()) { Shard shard = e.getKey(); Future<?> future = e.getValue(); @@ -154,10 +155,17 @@ public class ShardCommandManager extends BaseCommandManager { } catch (InterruptedException ex) { throw new IOException(ex); } catch (ExecutionException ex) { - throw new IOException(ex.getCause()); + if (collector == null) { + collector = new ExceptionCollector(); + } + collector.add(ex.getCause()); + continue; } resultMap.put(shard, object); } + if (collector != null) { + throw collector; + } return resultMap; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java index 2e992d9..6047bfa 100644 --- a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java +++ b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java @@ -18,6 +18,7 @@ package org.apache.blur.command; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.File; import java.io.FileOutputStream; @@ -87,9 +88,10 @@ public class ShardCommandManagerTest { @Test public void testGetCommands() { Map<String, BigInteger> commands = _manager.getCommands(); - assertEquals(1, commands.size()); + assertEquals(2, commands.size()); assertTrue(commands.containsKey("wait")); - assertEquals(BigInteger.ZERO, commands.get("wait")); + assertTrue(commands.containsKey("error")); + assertEquals(BigInteger.ZERO, commands.get("wait")); } @Test @@ -105,15 +107,18 @@ public class ShardCommandManagerTest { } @Test - public void testNewCommandLoading() throws IOException, TimeoutException, InterruptedException { + public void testNewCommandLoading() throws IOException, TimeoutException, InterruptedException, ExceptionCollector { _manager.close(); new File(_tmpPath).mkdirs(); File commandPath = new File(_commandPath); rmr(commandPath); + if (commandPath.exists()) { + fail("Command path [" + commandPath + "] still exists."); + } commandPath.mkdirs(); { InputStream inputStream = getClass().getResourceAsStream("/org/apache/blur/command/test1/test1.jar"); - File dest = new File(commandPath, "test.jar"); + File dest = new File(commandPath, "test1.jar"); FileOutputStream output = new FileOutputStream(dest); IOUtils.copy(inputStream, output); inputStream.close(); @@ -133,13 +138,14 @@ public class ShardCommandManagerTest { { InputStream inputStream = getClass().getResourceAsStream("/org/apache/blur/command/test2/test2.jar"); - File dest = new File(commandPath, "test.jar"); + File dest = new File(commandPath, "test2.jar"); FileOutputStream output = new FileOutputStream(dest); IOUtils.copy(inputStream, output); inputStream.close(); output.close(); } - manager.commandRefresh(); + + assertEquals(1, manager.commandRefresh()); { Args args = new Args(); @@ -168,7 +174,7 @@ public class ShardCommandManagerTest { } @Test - public void testShardCommandManagerNormalWait() throws IOException, TimeoutException { + public void testShardCommandManagerNormalWait() throws IOException, TimeoutException, ExceptionCollector { Response response; ExecutionId executionId = null; @@ -193,7 +199,23 @@ public class ShardCommandManagerTest { } @Test - public void testShardCommandManagerNormalWithCancel() throws IOException, TimeoutException { + public void testShardCommandManagerErrorWait() throws IOException, TimeoutException, ExceptionCollector { + Args args = new Args(); + args.set("seconds", 1); + args.set("table", "test"); + TableContextFactory tableContextFactory = getTableContextFactory(); + try { + _manager.execute(tableContextFactory, "error", args); + fail(); + } catch (ExceptionCollector e) { + Throwable t = e.getCause(); + assertTrue(t instanceof RuntimeException); + assertEquals("error-test", t.getMessage()); + } + } + + @Test + public void testShardCommandManagerNormalWithCancel() throws IOException, TimeoutException, ExceptionCollector { Response response; ExecutionId executionId = null; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-core/src/test/java/org/apache/blur/command/WaitForSeconds.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/command/WaitForSeconds.java b/blur-core/src/test/java/org/apache/blur/command/WaitForSeconds.java index 32ce8a0..15476f5 100644 --- a/blur-core/src/test/java/org/apache/blur/command/WaitForSeconds.java +++ b/blur-core/src/test/java/org/apache/blur/command/WaitForSeconds.java @@ -19,21 +19,11 @@ package org.apache.blur.command; import java.io.IOException; import java.util.concurrent.TimeUnit; -import org.apache.blur.command.Command; -import org.apache.blur.command.IndexContext; -import org.apache.blur.command.IndexReadCommand; import org.apache.blur.command.annotation.Argument; -import org.apache.blur.command.annotation.RequiredArguments; import org.apache.blur.command.annotation.OptionalArguments; @SuppressWarnings("serial") -@RequiredArguments({ @Argument(name = "table", value = "The name of the table to execute the wait for N number of seconds command.", type = String.class) }) -@OptionalArguments({ - - @Argument(name = "shard", value = "The shard id to execute the wait for N number of seconds command.", type = String.class), - @Argument(name = "seconds", value = "The number of seconds to sleep, the default is 30 seconds.", type = Integer.class) - -}) +@OptionalArguments({ @Argument(name = "seconds", value = "The number of seconds to sleep, the default is 30 seconds.", type = Integer.class) }) public class WaitForSeconds extends Command implements IndexReadCommand<Boolean> { @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-core/src/test/resources/META-INF/services/org.apache.blur.command.Commands ---------------------------------------------------------------------- diff --git a/blur-core/src/test/resources/META-INF/services/org.apache.blur.command.Commands b/blur-core/src/test/resources/META-INF/services/org.apache.blur.command.Commands index 51c8777..f06a262 100644 --- a/blur-core/src/test/resources/META-INF/services/org.apache.blur.command.Commands +++ b/blur-core/src/test/resources/META-INF/services/org.apache.blur.command.Commands @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -org.apache.blur.command.WaitForSeconds \ No newline at end of file +org.apache.blur.command.WaitForSeconds +org.apache.blur.command.ThrowException \ No newline at end of file
