Adding documentation to the commands.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/c2c78d55 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/c2c78d55 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/c2c78d55 Branch: refs/heads/master Commit: c2c78d553ff3f7f66c233c9f739e9677375f45c7 Parents: 90bcb15 Author: Aaron McCurry <[email protected]> Authored: Tue Sep 23 08:03:58 2014 -0400 Committer: Aaron McCurry <[email protected]> Committed: Tue Sep 23 08:03:58 2014 -0400 ---------------------------------------------------------------------- .../org/apache/blur/command/DocumentCount.java | 2 + .../blur/command/DocumentCountCombiner.java | 2 + .../blur/command/DocumentCountNoCombine.java | 2 + .../apache/blur/command/BaseCommandManager.java | 88 +++++++++++++++++--- .../blur/command/annotation/Description.java | 26 ++++++ .../apache/blur/server/FilteredBlurServer.java | 6 ++ .../blur/thrift/BlurControllerServer.java | 15 ++++ .../org/apache/blur/thrift/BlurShardServer.java | 14 ++++ .../java/org/apache/blur/thrift/TableAdmin.java | 22 +++++ .../org/apache/blur/thrift/ThriftServer.java | 10 ++- .../blur/command/ShardCommandManagerTest.java | 2 +- 11 files changed, 174 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c2c78d55/blur-command/src/main/java/org/apache/blur/command/DocumentCount.java ---------------------------------------------------------------------- diff --git a/blur-command/src/main/java/org/apache/blur/command/DocumentCount.java b/blur-command/src/main/java/org/apache/blur/command/DocumentCount.java index 03c2bad..ecf5b36 100644 --- a/blur-command/src/main/java/org/apache/blur/command/DocumentCount.java +++ b/blur-command/src/main/java/org/apache/blur/command/DocumentCount.java @@ -22,10 +22,12 @@ import org.apache.blur.command.Command; import org.apache.blur.command.IndexContext; import org.apache.blur.command.IndexReadCommand; import org.apache.blur.command.annotation.Argument; +import org.apache.blur.command.annotation.Description; import org.apache.blur.command.annotation.RequiredArguments; import org.apache.blur.command.annotation.OptionalArguments; @SuppressWarnings("serial") +@Description("Gets the number of visible documents in the index.") @RequiredArguments({ @Argument(name = "table", value = "The name of the table to execute the document count command.") }) @OptionalArguments({ @Argument(name = "shard", value = "The shard id to execute the document count command.") }) public class DocumentCount extends Command implements IndexReadCommand<Integer> { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c2c78d55/blur-command/src/main/java/org/apache/blur/command/DocumentCountCombiner.java ---------------------------------------------------------------------- diff --git a/blur-command/src/main/java/org/apache/blur/command/DocumentCountCombiner.java b/blur-command/src/main/java/org/apache/blur/command/DocumentCountCombiner.java index decea2e..e0db579 100644 --- a/blur-command/src/main/java/org/apache/blur/command/DocumentCountCombiner.java +++ b/blur-command/src/main/java/org/apache/blur/command/DocumentCountCombiner.java @@ -28,10 +28,12 @@ import org.apache.blur.command.IndexReadCombiningCommand; import org.apache.blur.command.Server; import org.apache.blur.command.Shard; import org.apache.blur.command.annotation.Argument; +import org.apache.blur.command.annotation.Description; import org.apache.blur.command.annotation.RequiredArguments; import org.apache.blur.command.annotation.OptionalArguments; @SuppressWarnings("serial") +@Description("Gets the number of visible documents in the index.") @RequiredArguments({ @Argument(name = "table", value = "The name of the table to execute the document count command.") }) @OptionalArguments({ @Argument(name = "shard", value = "The shard id to execute the document count command.") }) public class DocumentCountCombiner extends Command implements ClusterCommand<Long>, http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c2c78d55/blur-command/src/main/java/org/apache/blur/command/DocumentCountNoCombine.java ---------------------------------------------------------------------- diff --git a/blur-command/src/main/java/org/apache/blur/command/DocumentCountNoCombine.java b/blur-command/src/main/java/org/apache/blur/command/DocumentCountNoCombine.java index 8c3a010..0ddd045 100644 --- a/blur-command/src/main/java/org/apache/blur/command/DocumentCountNoCombine.java +++ b/blur-command/src/main/java/org/apache/blur/command/DocumentCountNoCombine.java @@ -27,10 +27,12 @@ import org.apache.blur.command.IndexContext; import org.apache.blur.command.IndexReadCommand; import org.apache.blur.command.Shard; import org.apache.blur.command.annotation.Argument; +import org.apache.blur.command.annotation.Description; import org.apache.blur.command.annotation.RequiredArguments; import org.apache.blur.command.annotation.OptionalArguments; @SuppressWarnings("serial") +@Description("Gets the number of visible documents in the index.") @RequiredArguments({ @Argument(name = "table", value = "The name of the table to execute the document count command.") }) @OptionalArguments({ @Argument(name = "shard", value = "The shard id to execute the document count command.") }) public class DocumentCountNoCombine extends Command implements IndexReadCommand<Integer>, ClusterCommand<Long> { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c2c78d55/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java b/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java index 8c13b94..514932d 100644 --- a/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java +++ b/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java @@ -5,6 +5,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Method; import java.math.BigInteger; import java.net.MalformedURLException; import java.net.URL; @@ -31,6 +32,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.apache.blur.command.annotation.Argument; +import org.apache.blur.command.annotation.Description; import org.apache.blur.command.annotation.OptionalArguments; import org.apache.blur.command.annotation.RequiredArguments; import org.apache.blur.concurrent.Executors; @@ -115,7 +117,7 @@ public class BaseCommandManager implements Closeable { return getArguments(commandName, true); } - private Map<String, String> getArguments(String commandName, boolean optional) { + protected Map<String, String> getArguments(String commandName, boolean optional) { Command command = _command.get(commandName); if (command == null) { return null; @@ -123,19 +125,28 @@ public class BaseCommandManager implements Closeable { Class<? extends Command> clazz = command.getClass(); Map<String, String> arguments = new TreeMap<String, String>(); Argument[] args = getArgumentArray(clazz, optional); + if (args == null) { + return arguments; + } for (Argument argument : args) { arguments.put(argument.name(), argument.value()); } return arguments; } - private Argument[] getArgumentArray(Class<? extends Command> clazz, boolean optional) { + protected Argument[] getArgumentArray(Class<? extends Command> clazz, boolean optional) { if (optional) { - OptionalArguments requiredArguments = clazz.getAnnotation(OptionalArguments.class); - return requiredArguments.value(); + OptionalArguments arguments = clazz.getAnnotation(OptionalArguments.class); + if (arguments == null) { + return null; + } + return arguments.value(); } else { - RequiredArguments requiredArguments = clazz.getAnnotation(RequiredArguments.class); - return requiredArguments.value(); + RequiredArguments arguments = clazz.getAnnotation(RequiredArguments.class); + if (arguments == null) { + return null; + } + return arguments.value(); } } @@ -181,7 +192,7 @@ public class BaseCommandManager implements Closeable { } LOG.info("Copying new command with hash [{2}] set from [{0}] into [{1}].", fileStatus.getPath(), file.getAbsolutePath(), hashOfContents.toString(Character.MAX_RADIX)); - copyLocal(fileSystem, fileStatus.getPath(), file); + copyLocal(fileSystem, fileStatus, file); URLClassLoader loader = new URLClassLoader(getUrls(file).toArray(new URL[] {})); Enumeration<URL> resources = loader.getResources(META_INF_SERVICES_ORG_APACHE_BLUR_COMMAND_COMMANDS); loadCommandClasses(resources, loader, hashOfContents); @@ -201,16 +212,17 @@ public class BaseCommandManager implements Closeable { return urls; } - protected void copyLocal(FileSystem fileSystem, Path path, File destDir) throws IOException { + protected void copyLocal(FileSystem fileSystem, FileStatus fileStatus, File destDir) throws IOException { + Path path = fileStatus.getPath(); File file = new File(destDir, path.getName()); - if (fileSystem.isDirectory(path)) { + if (fileStatus.isDir()) { if (!file.mkdirs()) { LOG.error("Error while trying to create a sub directory [{0}].", file.getAbsolutePath()); throw new IOException("Error while trying to create a sub directory [" + file.getAbsolutePath() + "]."); } FileStatus[] listStatus = fileSystem.listStatus(path); - for (FileStatus fileStatus : listStatus) { - copyLocal(fileSystem, fileStatus.getPath(), file); + for (FileStatus fs : listStatus) { + copyLocal(fileSystem, fs, file); } } else { FileOutputStream output = new FileOutputStream(file); @@ -222,7 +234,7 @@ public class BaseCommandManager implements Closeable { } protected BigInteger checkContents(FileStatus fileStatus, FileSystem fileSystem) throws IOException { - if (fileStatus.isDirectory()) { + if (fileStatus.isDir()) { BigInteger count = BigInteger.ZERO; Path path = fileStatus.getPath(); FileStatus[] listStatus = fileSystem.listStatus(path); @@ -395,4 +407,56 @@ public class BaseCommandManager implements Closeable { } return tables; } + + public String getDescription(String commandName) { + Command command = _command.get(commandName); + if (command == null) { + return null; + } + Class<? extends Command> clazz = command.getClass(); + Description description = clazz.getAnnotation(Description.class); + if (description == null) { + return null; + } + return description.value(); + } + + public String getReturnType(String commandName) { + Command command = _command.get(commandName); + if (command == null) { + return null; + } + + String shardServerReturn; + try { + if (command instanceof IndexReadCommand) { + IndexReadCommand<?> indexReadCommand = (IndexReadCommand<?>) command; + Method method = indexReadCommand.getClass().getMethod("execute", new Class[] { IndexContext.class }); + Class<?> returnType = method.getReturnType(); + shardServerReturn = "shard->" + returnType.getSimpleName(); + } else if (command instanceof IndexReadCombiningCommand) { + IndexReadCombiningCommand<?, ?> indexReadCombiningCommand = (IndexReadCombiningCommand<?, ?>) command; + Method method = indexReadCombiningCommand.getClass().getMethod("combine", new Class[] { Map.class }); + Class<?> returnType = method.getReturnType(); + shardServerReturn = "server->" + returnType.getSimpleName(); + } else { + shardServerReturn = null; + } + if (command instanceof ClusterCommand) { + ClusterCommand<?> clusterCommand = (ClusterCommand<?>) command; + Method method = clusterCommand.getClass().getMethod("clusterExecute", new Class[] { Map.class }); + Class<?> returnType = method.getReturnType(); + String clusterReturn = "cluster->" + returnType.getSimpleName(); + if (shardServerReturn == null) { + return clusterReturn; + } else { + return clusterReturn + "," + shardServerReturn; + } + } + return shardServerReturn; + } catch (Exception e) { + throw new RuntimeException("Unknown error while trying to get return type.", e); + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c2c78d55/blur-core/src/main/java/org/apache/blur/command/annotation/Description.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/annotation/Description.java b/blur-core/src/main/java/org/apache/blur/command/annotation/Description.java new file mode 100644 index 0000000..7d93568 --- /dev/null +++ b/blur-core/src/main/java/org/apache/blur/command/annotation/Description.java @@ -0,0 +1,26 @@ +package org.apache.blur.command.annotation; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +@Retention(RetentionPolicy.RUNTIME) +public @interface Description { + String value(); +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c2c78d55/blur-core/src/main/java/org/apache/blur/server/FilteredBlurServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/FilteredBlurServer.java b/blur-core/src/main/java/org/apache/blur/server/FilteredBlurServer.java index 6c5106a..2154916 100644 --- a/blur-core/src/main/java/org/apache/blur/server/FilteredBlurServer.java +++ b/blur-core/src/main/java/org/apache/blur/server/FilteredBlurServer.java @@ -29,6 +29,7 @@ import org.apache.blur.thrift.generated.BlurQuery; import org.apache.blur.thrift.generated.BlurQueryStatus; import org.apache.blur.thrift.generated.BlurResults; import org.apache.blur.thrift.generated.ColumnDefinition; +import org.apache.blur.thrift.generated.CommandDescriptor; import org.apache.blur.thrift.generated.CommandStatus; import org.apache.blur.thrift.generated.CommandStatusState; import org.apache.blur.thrift.generated.FetchResult; @@ -280,4 +281,9 @@ public class FilteredBlurServer implements Iface { _iface.commandCancel(executionId); } + @Override + public List<CommandDescriptor> listInstalledCommands() throws BlurException, TException { + return _iface.listInstalledCommands(); + } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c2c78d55/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java index c3434994..ad10ac0 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java @@ -87,6 +87,7 @@ import org.apache.blur.thrift.generated.BlurQueryStatus; import org.apache.blur.thrift.generated.BlurResult; import org.apache.blur.thrift.generated.BlurResults; import org.apache.blur.thrift.generated.ColumnDefinition; +import org.apache.blur.thrift.generated.CommandDescriptor; import org.apache.blur.thrift.generated.CommandStatus; import org.apache.blur.thrift.generated.CommandStatusState; import org.apache.blur.thrift.generated.ErrorType; @@ -1640,6 +1641,19 @@ public class BlurControllerServer extends TableAdmin implements Iface { } @Override + public List<CommandDescriptor> listInstalledCommands() throws BlurException, TException { + try { + return listInstalledCommands(_commandManager); + } catch (Exception e) { + LOG.error("Unknown error while trying to get a list of installed commands [{0}]", e); + if (e instanceof BlurException) { + throw (BlurException) e; + } + throw new BException(e.getMessage(), e); + } + } + + @Override public void refresh() throws TException { // This is a NO-OP at this point for the controller. } @@ -1659,4 +1673,5 @@ public class BlurControllerServer extends TableAdmin implements Iface { public void commandCancel(String executionId) throws BlurException, TException { throw new BException("Not Implemented"); } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c2c78d55/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java index 00c7509..c438fa8 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java @@ -53,6 +53,7 @@ import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.BlurQuery; import org.apache.blur.thrift.generated.BlurQueryStatus; import org.apache.blur.thrift.generated.BlurResults; +import org.apache.blur.thrift.generated.CommandDescriptor; import org.apache.blur.thrift.generated.CommandStatus; import org.apache.blur.thrift.generated.CommandStatusState; import org.apache.blur.thrift.generated.FetchResult; @@ -646,6 +647,19 @@ public class BlurShardServer extends TableAdmin implements Iface { public void refresh() throws TException { ShardServerContext.resetSearchers(); } + + @Override + public List<CommandDescriptor> listInstalledCommands() throws BlurException, TException { + try { + return listInstalledCommands(_commandManager); + } catch (Exception e) { + LOG.error("Unknown error while trying to get a list of installed commands [{0}]", e); + if (e instanceof BlurException) { + throw (BlurException) e; + } + throw new BException(e.getMessage(), e); + } + } @Override public List<String> commandStatusList(int startingAt, short fetch, CommandStatusState state) throws BlurException, http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c2c78d55/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java index 95afda7..f396232 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java @@ -18,19 +18,23 @@ package org.apache.blur.thrift; */ import java.io.File; import java.io.IOException; +import java.math.BigInteger; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.TreeMap; import javax.xml.parsers.FactoryConfigurationError; import org.apache.blur.BlurConfiguration; import org.apache.blur.analysis.FieldManager; import org.apache.blur.analysis.FieldTypeDefinition; +import org.apache.blur.command.BaseCommandManager; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.manager.clusterstatus.ClusterStatus; @@ -39,6 +43,7 @@ import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.ColumnDefinition; +import org.apache.blur.thrift.generated.CommandDescriptor; import org.apache.blur.thrift.generated.Level; import org.apache.blur.thrift.generated.Metric; import org.apache.blur.thrift.generated.Schema; @@ -616,6 +621,23 @@ public abstract class TableAdmin implements Iface { } } + protected List<CommandDescriptor> listInstalledCommands(BaseCommandManager commandManager) { + List<CommandDescriptor> result = new ArrayList<CommandDescriptor>(); + Map<String, BigInteger> commands = new TreeMap<String, BigInteger>(commandManager.getCommands()); + for (Entry<String, BigInteger> e : commands.entrySet()) { + CommandDescriptor commandDescriptor = new CommandDescriptor(); + String commandName = e.getKey(); + commandDescriptor.setCommandName(commandName); + commandDescriptor.setVersion(e.getValue().toString(Character.MAX_RADIX)); + commandDescriptor.setDescription(commandManager.getDescription(commandName)); + commandDescriptor.setOptionalArguments(commandManager.getOptionalArguments(commandName)); + commandDescriptor.setRequiredArguments(commandManager.getRequiredArguments(commandName)); + commandDescriptor.setReturnType(commandManager.getReturnType(commandName)); + result.add(commandDescriptor); + } + return result; + } + private void reloadConfig() throws MalformedURLException, FactoryConfigurationError, BException { String blurHome = System.getenv("BLUR_HOME"); if (blurHome != null) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c2c78d55/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java b/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java index 5a5b32e..c961959 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/ThriftServer.java @@ -40,6 +40,7 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.UnknownHostException; +import java.util.UUID; import java.util.concurrent.ExecutorService; import org.apache.blur.BlurConfiguration; @@ -127,15 +128,20 @@ public class ThriftServer { LOG.info("Attempting to use default tmp directory [{0}]", tmp); } } - if (!tmp.mkdirs()) { + if (!tmp.exists() && !tmp.mkdirs()) { throw new IOException("Cannot create tmp directory [" + tmp.toURI() + "], please create directory or configure property [" + propName + "]."); } + File file = new File(tmp, UUID.randomUUID().toString()); + if (!file.createNewFile()) { + throw new IOException("Cannot create tmp file in [" + tmp.toURI() + "]."); + } + file.delete(); return tmp.toURI().toString(); } private static File getTmpDir() { - return new File(System.getProperty("java.io.tmpdir")); + return new File(System.getProperty("java.io.tmpdir"), "blur_tmp"); } public static TraceStorage setupTraceStorage(BlurConfiguration configuration) throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c2c78d55/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java index 93016f4..2e992d9 100644 --- a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java +++ b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java @@ -89,7 +89,7 @@ public class ShardCommandManagerTest { Map<String, BigInteger> commands = _manager.getCommands(); assertEquals(1, commands.size()); assertTrue(commands.containsKey("wait")); - assertEquals(BigInteger.ZERO, commands.get("wait")); + assertEquals(BigInteger.ZERO, commands.get("wait")); } @Test
