Fixing the argument descriptions, returntype and shell.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/546d2de8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/546d2de8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/546d2de8 Branch: refs/heads/master Commit: 546d2de83abb27e31bec7ead84bd6fbaf8d65d38 Parents: fb44fa5 Author: Aaron McCurry <[email protected]> Authored: Wed Oct 1 09:56:29 2014 -0400 Committer: Aaron McCurry <[email protected]> Committed: Wed Oct 1 09:56:29 2014 -0400 ---------------------------------------------------------------------- .../apache/blur/command/BaseCommandManager.java | 47 +---- .../java/org/apache/blur/command/Command.java | 67 ++++++- .../commandtype/ClusterExecuteCommand.java | 15 +- .../ClusterExecuteServerReadCommand.java | 12 ++ .../commandtype/ClusterIndexReadCommand.java | 12 ++ .../commandtype/ClusterServerReadCommand.java | 16 +- .../command/commandtype/IndexReadCommand.java | 12 ++ .../command/commandtype/ServerReadCommand.java | 15 +- .../java/org/apache/blur/thrift/TableAdmin.java | 15 +- .../blur/command/ShardCommandManagerTest.java | 14 +- .../mapreduce/lib/GenericBlurRecordWriter.java | 2 +- .../org/apache/blur/mapreduce/lib/IOUtil.java | 3 + .../mapreduce/lib/v2/DirectIndexingDriver.java | 174 ++++++++++++++++++- .../shell/DescribePlatformCommandCommand.java | 23 ++- 14 files changed, 360 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java b/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java index 4779013..e41a32b 100644 --- a/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java +++ b/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java @@ -5,7 +5,6 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import java.lang.reflect.Method; import java.math.BigInteger; import java.net.MalformedURLException; import java.net.URL; @@ -30,9 +29,6 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.apache.blur.command.annotation.Description; -import org.apache.blur.command.commandtype.ClusterExecuteServerReadCommand; -import org.apache.blur.command.commandtype.ServerReadCommand; -import org.apache.blur.command.commandtype.IndexReadCommand; import org.apache.blur.concurrent.Executors; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; @@ -108,12 +104,14 @@ public abstract class BaseCommandManager implements Closeable { return new HashMap<String, BigInteger>(_commandLoadTime); } - public Map<String, String> getRequiredArguments(String commandName) { - return getArguments(commandName, false); + public Set<Argument> getRequiredArguments(String commandName) { + Command<?> command = getCommandObject(commandName, null); + return command.getRequiredArguments(); } - public Map<String, String> getOptionalArguments(String commandName) { - return getArguments(commandName, true); + public Set<Argument> getOptionalArguments(String commandName) { + Command<?> command = getCommandObject(commandName, null); + return command.getOptionalArguments(); } protected Map<String, String> getArguments(String commandName, boolean optional) { @@ -481,38 +479,7 @@ public abstract class BaseCommandManager implements Closeable { if (command == null) { return null; } - - String shardServerReturn; - try { - if (command instanceof IndexReadCommand) { - IndexReadCommand<?> indexReadCommand = (IndexReadCommand<?>) command; - Method method = indexReadCommand.getClass().getMethod("execute", new Class[] { IndexContext.class }); - Class<?> returnType = method.getReturnType(); - shardServerReturn = "shard->(" + returnType.getSimpleName() + ")"; - } else if (command instanceof ServerReadCommand) { - ServerReadCommand<?, ?> indexReadCombiningCommand = (ServerReadCommand<?, ?>) command; - Method method = indexReadCombiningCommand.getClass().getMethod("combine", - new Class[] { CombiningContext.class, Map.class }); - Class<?> returnType = method.getReturnType(); - shardServerReturn = "server->(" + returnType.getSimpleName() + ")"; - } else { - shardServerReturn = null; - } - if (command instanceof ClusterExecuteServerReadCommand) { - ClusterExecuteServerReadCommand<?> clusterCommand = (ClusterExecuteServerReadCommand<?>) command; - Method method = clusterCommand.getClass().getMethod("clusterExecute", new Class[] { Map.class }); - Class<?> returnType = method.getReturnType(); - String clusterReturn = "cluster->(" + returnType.getSimpleName() + ")"; - if (shardServerReturn == null) { - return clusterReturn; - } else { - return clusterReturn + "," + shardServerReturn; - } - } - return shardServerReturn; - } catch (Exception e) { - throw new RuntimeException("Unknown error while trying to get return type.", e); - } + return command.getReturnType(); } protected Arguments toArguments(Command<?> command) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/command/Command.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/Command.java b/blur-core/src/main/java/org/apache/blur/command/Command.java index f6f7314..89b6e0e 100644 --- a/blur-core/src/main/java/org/apache/blur/command/Command.java +++ b/blur-core/src/main/java/org/apache/blur/command/Command.java @@ -17,9 +17,15 @@ package org.apache.blur.command; import java.io.IOException; +import java.lang.annotation.Annotation; +import java.lang.reflect.Field; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; import java.util.HashSet; import java.util.Set; +import org.apache.blur.command.annotation.OptionalArgument; +import org.apache.blur.command.annotation.RequiredArgument; import org.apache.blur.thrift.generated.Blur.Iface; public abstract class Command<R> implements Cloneable { @@ -45,11 +51,70 @@ public abstract class Command<R> implements Cloneable { throw new RuntimeException(e); } } - + public static <T> Set<T> asSet(T t) { Set<T> set = new HashSet<T>(); set.add(t); return set; } + public Set<Argument> getRequiredArguments() { + Set<Argument> arguments = new HashSet<Argument>(); + readArguments(getClass(), arguments, RequiredArgument.class); + return arguments; + } + + public Set<Argument> getOptionalArguments() { + Set<Argument> arguments = new HashSet<Argument>(); + readArguments(getClass(), arguments, OptionalArgument.class); + return arguments; + } + + public abstract String getReturnType(); + + private void readArguments(Class<?> clazz, Set<Argument> arguments, Class<? extends Annotation> annotationClass) { + if (clazz.equals(Command.class)) { + return; + } + readArguments(clazz.getSuperclass(), arguments, annotationClass); + Field[] declaredFields = clazz.getDeclaredFields(); + for (Field field : declaredFields) { + field.setAccessible(true); + Annotation annotation = field.getAnnotation(annotationClass); + Class<?> type = field.getType(); + Type genericType = field.getGenericType(); + Type[] actualTypeArguments = null; + if (genericType instanceof ParameterizedType) { + ParameterizedType parameterizedType = (ParameterizedType) genericType; + actualTypeArguments = parameterizedType.getActualTypeArguments(); + } + String simpleName = getTypeName(type.getSimpleName(), actualTypeArguments); + String name = field.getName(); + if (annotation instanceof RequiredArgument) { + RequiredArgument requiredArgument = (RequiredArgument) annotation; + String value = requiredArgument.value(); + arguments.add(new Argument(name, simpleName, value)); + } else if (annotation instanceof OptionalArgument) { + OptionalArgument optionalArgument = (OptionalArgument) annotation; + String value = optionalArgument.value(); + arguments.add(new Argument(name, simpleName, value)); + } + } + } + + private String getTypeName(String simpleName, Type[] actualTypeArguments) { + if (actualTypeArguments == null) { + return simpleName; + } + StringBuilder builder = new StringBuilder(); + builder.append(simpleName).append('<'); + for (Type type : actualTypeArguments) { + if (type instanceof Class) { + builder.append(((Class<?>) type).getSimpleName()); + } + } + builder.append('>'); + return builder.toString(); + } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteCommand.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteCommand.java b/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteCommand.java index 5ba0863..4312169 100644 --- a/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteCommand.java +++ b/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteCommand.java @@ -1,13 +1,14 @@ package org.apache.blur.command.commandtype; import java.io.IOException; +import java.lang.reflect.Method; import org.apache.blur.command.ClusterContext; import org.apache.blur.command.Command; import org.apache.blur.command.CommandRunner; import org.apache.blur.thirdparty.thrift_0_9_0.TException; -import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.Blur.Iface; +import org.apache.blur.thrift.generated.BlurException; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -62,4 +63,16 @@ public abstract class ClusterExecuteCommand<T> extends Command<T> { throw new IOException(e); } } + + @Override + public String getReturnType() { + try { + Method method = getClass().getMethod("clusterExecute", new Class[] { ClusterContext.class }); + Class<?> returnType = method.getReturnType(); + return returnType.getSimpleName(); + } catch (Exception e) { + throw new RuntimeException("Unknown error while trying to get return type.", e); + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteServerReadCommand.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteServerReadCommand.java b/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteServerReadCommand.java index 58d206b..4a7c986 100644 --- a/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteServerReadCommand.java +++ b/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteServerReadCommand.java @@ -17,6 +17,7 @@ package org.apache.blur.command.commandtype; import java.io.IOException; +import java.lang.reflect.Method; import java.util.Map; import org.apache.blur.command.ClusterContext; @@ -38,6 +39,17 @@ public abstract class ClusterExecuteServerReadCommand<T> extends Command<T> impl InterruptedException; public abstract T clusterExecute(ClusterContext context) throws IOException, InterruptedException; + + @Override + public String getReturnType() { + try { + Method method = getClass().getMethod("clusterExecute", new Class[] { ClusterContext.class }); + Class<?> returnType = method.getReturnType(); + return returnType.getSimpleName(); + } catch (Exception e) { + throw new RuntimeException("Unknown error while trying to get return type.", e); + } + } @Override public T run() throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterIndexReadCommand.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterIndexReadCommand.java b/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterIndexReadCommand.java index ad526d2..83130e7 100644 --- a/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterIndexReadCommand.java +++ b/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterIndexReadCommand.java @@ -17,6 +17,7 @@ package org.apache.blur.command.commandtype; import java.io.IOException; +import java.lang.reflect.Method; import org.apache.blur.command.ClusterContext; import org.apache.blur.command.Command; @@ -34,6 +35,17 @@ public abstract class ClusterIndexReadCommand<T1, T2> extends Command<T2> implem public abstract T2 clusterExecute(ClusterContext context) throws IOException, InterruptedException; @Override + public String getReturnType() { + try { + Method method = getClass().getMethod("clusterExecute", new Class[] { ClusterContext.class }); + Class<?> returnType = method.getReturnType(); + return returnType.getSimpleName(); + } catch (Exception e) { + throw new RuntimeException("Unknown error while trying to get return type.", e); + } + } + + @Override public T2 run() throws IOException { try { return CommandRunner.run(this); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterServerReadCommand.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterServerReadCommand.java b/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterServerReadCommand.java index ec582f7..be7c0eb 100644 --- a/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterServerReadCommand.java +++ b/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterServerReadCommand.java @@ -17,17 +17,18 @@ package org.apache.blur.command.commandtype; import java.io.IOException; +import java.lang.reflect.Method; import java.util.Map; import org.apache.blur.command.CombiningContext; import org.apache.blur.command.Command; import org.apache.blur.command.CommandRunner; import org.apache.blur.command.IndexContext; -import org.apache.blur.command.ServerRead; import org.apache.blur.command.Location; +import org.apache.blur.command.ServerRead; import org.apache.blur.thirdparty.thrift_0_9_0.TException; -import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.Blur.Iface; +import org.apache.blur.thrift.generated.BlurException; public abstract class ClusterServerReadCommand<T> extends Command<T> implements ServerRead<T, T> { @@ -37,6 +38,17 @@ public abstract class ClusterServerReadCommand<T> extends Command<T> implements InterruptedException; @Override + public String getReturnType() { + try { + Method method = getClass().getMethod("combine", new Class[] { CombiningContext.class, Map.class }); + Class<?> returnType = method.getReturnType(); + return returnType.getSimpleName(); + } catch (Exception e) { + throw new RuntimeException("Unknown error while trying to get return type.", e); + } + } + + @Override public T run() throws IOException { try { return CommandRunner.run(this); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/command/commandtype/IndexReadCommand.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/commandtype/IndexReadCommand.java b/blur-core/src/main/java/org/apache/blur/command/commandtype/IndexReadCommand.java index 5d2b75d..c0343ea 100644 --- a/blur-core/src/main/java/org/apache/blur/command/commandtype/IndexReadCommand.java +++ b/blur-core/src/main/java/org/apache/blur/command/commandtype/IndexReadCommand.java @@ -17,6 +17,7 @@ package org.apache.blur.command.commandtype; import java.io.IOException; +import java.lang.reflect.Method; import java.util.Map; import org.apache.blur.command.Command; @@ -33,6 +34,17 @@ public abstract class IndexReadCommand<T> extends Command<Map<Shard, T>> impleme public abstract T execute(IndexContext context) throws IOException, InterruptedException; @Override + public String getReturnType() { + try { + Method method = getClass().getMethod("execute", new Class[] { IndexContext.class }); + Class<?> returnType = method.getReturnType(); + return "map(Shard," + returnType.getSimpleName() + ")"; + } catch (Exception e) { + throw new RuntimeException("Unknown error while trying to get return type.", e); + } + } + + @Override public Map<Shard, T> run() throws IOException { try { return CommandRunner.run(this); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/command/commandtype/ServerReadCommand.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/commandtype/ServerReadCommand.java b/blur-core/src/main/java/org/apache/blur/command/commandtype/ServerReadCommand.java index 7a553e9..0523408 100644 --- a/blur-core/src/main/java/org/apache/blur/command/commandtype/ServerReadCommand.java +++ b/blur-core/src/main/java/org/apache/blur/command/commandtype/ServerReadCommand.java @@ -17,6 +17,7 @@ package org.apache.blur.command.commandtype; import java.io.IOException; +import java.lang.reflect.Method; import java.util.Map; import org.apache.blur.command.CombiningContext; @@ -30,8 +31,7 @@ import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.Blur.Iface; -public abstract class ServerReadCommand<T1, T2> extends Command<Map<Server, T2>> implements - ServerRead<T1, T2> { +public abstract class ServerReadCommand<T1, T2> extends Command<Map<Server, T2>> implements ServerRead<T1, T2> { public abstract T1 execute(IndexContext context) throws IOException, InterruptedException; @@ -39,6 +39,17 @@ public abstract class ServerReadCommand<T1, T2> extends Command<Map<Server, T2>> InterruptedException; @Override + public String getReturnType() { + try { + Method method = getClass().getMethod("combine", new Class[] { CombiningContext.class, Map.class }); + Class<?> returnType = method.getReturnType(); + return "map(Server," + returnType.getSimpleName() + ")"; + } catch (Exception e) { + throw new RuntimeException("Unknown error while trying to get return type.", e); + } + } + + @Override public Map<Server, T2> run() throws IOException { try { return CommandRunner.run(this); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java index f396232..d221774 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java @@ -34,12 +34,14 @@ import javax.xml.parsers.FactoryConfigurationError; import org.apache.blur.BlurConfiguration; import org.apache.blur.analysis.FieldManager; import org.apache.blur.analysis.FieldTypeDefinition; +import org.apache.blur.command.Argument; import org.apache.blur.command.BaseCommandManager; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.manager.clusterstatus.ClusterStatus; import org.apache.blur.server.TableContext; import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thrift.generated.ArgumentDescriptor; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.ColumnDefinition; @@ -630,14 +632,23 @@ public abstract class TableAdmin implements Iface { commandDescriptor.setCommandName(commandName); commandDescriptor.setVersion(e.getValue().toString(Character.MAX_RADIX)); commandDescriptor.setDescription(commandManager.getDescription(commandName)); - commandDescriptor.setOptionalArguments(commandManager.getOptionalArguments(commandName)); - commandDescriptor.setRequiredArguments(commandManager.getRequiredArguments(commandName)); + commandDescriptor.setOptionalArguments(toThrift(commandManager.getOptionalArguments(commandName))); + commandDescriptor.setRequiredArguments(toThrift(commandManager.getRequiredArguments(commandName))); commandDescriptor.setReturnType(commandManager.getReturnType(commandName)); result.add(commandDescriptor); } return result; } + private Map<String, ArgumentDescriptor> toThrift(Set<Argument> arguments) { + Map<String, ArgumentDescriptor> result = new HashMap<String, ArgumentDescriptor>(); + for (Argument argument : arguments) { + String name = argument.getName(); + result.put(name, new ArgumentDescriptor(name, argument.getType(), argument.getDescription())); + } + return result; + } + private void reloadConfig() throws MalformedURLException, FactoryConfigurationError, BException { String blurHome = System.getenv("BLUR_HOME"); if (blurHome != null) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java index 8ca946e..f8ef2e5 100644 --- a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java +++ b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java @@ -28,6 +28,7 @@ import java.math.BigInteger; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.SortedSet; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -97,14 +98,17 @@ public class ShardCommandManagerTest { @Test public void testDocumentation() { - Map<String, String> requiredArgs = _manager.getRequiredArguments("wait"); - assertTrue(requiredArgs.containsKey("table")); + Set<Argument> requiredArgs = _manager.getRequiredArguments("wait"); assertEquals(1, requiredArgs.size()); + assertTrue(requiredArgs.contains(new Argument("table", "String", "The name of the table."))); - Map<String, String> optionalArgs = _manager.getOptionalArguments("wait"); - assertTrue(optionalArgs.containsKey("seconds")); - assertTrue(optionalArgs.containsKey("shard")); + Set<Argument> optionalArgs = _manager.getOptionalArguments("wait"); + System.out.println(optionalArgs); assertEquals(2, optionalArgs.size()); + assertTrue(optionalArgs.contains(new Argument("seconds", "int", + "The number of seconds to sleep, the default is 30 seconds."))); + assertTrue(optionalArgs.contains(new Argument("shards", "Set<String>", "The shard ids (e.g. shard-0000000)."))); + } @Test http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java index 4bd311b..8b90331 100644 --- a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java +++ b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java @@ -263,7 +263,7 @@ public class GenericBlurRecordWriter { _localTmpPath = null; } - private Record getRecord(BlurRecord value) { + public static Record getRecord(BlurRecord value) { Record record = new Record(); record.setRecordId(value.getRecordId()); record.setFamily(value.getFamily()); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/IOUtil.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/IOUtil.java b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/IOUtil.java index 46c030f..cf62cbc 100644 --- a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/IOUtil.java +++ b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/IOUtil.java @@ -20,6 +20,9 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import org.apache.blur.thrift.generated.Column; +import org.apache.blur.thrift.generated.Record; + public class IOUtil { public static final String UTF_8 = "UTF-8"; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/v2/DirectIndexingDriver.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/v2/DirectIndexingDriver.java b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/v2/DirectIndexingDriver.java index bd56003..5d17aef 100644 --- a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/v2/DirectIndexingDriver.java +++ b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/v2/DirectIndexingDriver.java @@ -17,8 +17,12 @@ package org.apache.blur.mapreduce.lib.v2; import java.io.IOException; +import java.util.List; import org.apache.blur.analysis.FieldManager; +import org.apache.blur.mapreduce.lib.BlurRecord; +import org.apache.blur.mapreduce.lib.GenericBlurRecordWriter; +import org.apache.blur.server.TableContext; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.tools.DFSAdmin; @@ -26,32 +30,188 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.Mapper.Context; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.IndexableFieldType; +import org.apache.lucene.util.IOUtils; public class DirectIndexingDriver implements Tool { - public static class DirectIndexingMapper extends - Mapper<IntWritable, DocumentWritable, LuceneKeyWritable, NullWritable> { + public static class DirectIndexingMapper extends Mapper<IntWritable, BlurRecord, LuceneKeyWritable, NullWritable> { private FieldManager _fieldManager; + private TableContext _tableContext; + private Analyzer _analyzer; + + // private fieldState.reset(); @Override protected void setup(Context context) throws IOException, InterruptedException { - + _fieldManager = _tableContext.getFieldManager(); + _analyzer = _fieldManager.getAnalyzerForIndex(); } @Override - protected void map(IntWritable key, DocumentWritable value, Context context) throws IOException, - InterruptedException { + protected void map(IntWritable key, BlurRecord record, Context context) throws IOException, InterruptedException { int documentId = key.get(); - + String rowId = record.getRowId(); + List<Field> fields = _fieldManager.getFields(rowId, GenericBlurRecordWriter.getRecord(record)); + // write(documentId, fields, context); } +// public void processFields(final IndexableField[] fields, final int count) throws IOException { +// +// fieldState.reset(); +// +// final boolean doInvert = consumer.start(fields, count); +// +// for (int i = 0; i < count; i++) { +// +// final IndexableField field = fields[i]; +// final IndexableFieldType fieldType = field.fieldType(); +// +// // TODO FI: this should be "genericized" to querying +// // consumer if it wants to see this particular field +// // tokenized. +// if (fieldType.indexed() && doInvert) { +// final boolean analyzed = fieldType.tokenized() && docState.analyzer != null; +// +// // if the field omits norms, the boost cannot be indexed. +// if (fieldType.omitNorms() && field.boost() != 1.0f) { +// throw new UnsupportedOperationException("You cannot set an index-time boost: norms are omitted for field '" +// + field.name() + "'"); +// } +// +// // only bother checking offsets if something will consume them. +// // TODO: after we fix analyzers, also check if termVectorOffsets will +// // be indexed. +// final boolean checkOffsets = fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS; +// int lastStartOffset = 0; +// +// if (i > 0) { +// fieldState.position += analyzed ? docState.analyzer.getPositionIncrementGap(fieldInfo.name) : 0; +// } +// +// final TokenStream stream = field.tokenStream(docState.analyzer); +// // reset the TokenStream to the first token +// stream.reset(); +// +// boolean success2 = false; +// +// try { +// boolean hasMoreTokens = stream.incrementToken(); +// +// fieldState.attributeSource = stream; +// +// OffsetAttribute offsetAttribute = fieldState.attributeSource.addAttribute(OffsetAttribute.class); +// PositionIncrementAttribute posIncrAttribute = fieldState.attributeSource +// .addAttribute(PositionIncrementAttribute.class); +// +// if (hasMoreTokens) { +// consumer.start(field); +// +// do { +// // If we hit an exception in stream.next below +// // (which is fairly common, eg if analyzer +// // chokes on a given document), then it's +// // non-aborting and (above) this one document +// // will be marked as deleted, but still +// // consume a docID +// +// final int posIncr = posIncrAttribute.getPositionIncrement(); +// if (posIncr < 0) { +// throw new IllegalArgumentException("position increment must be >=0 (got " + posIncr + ")"); +// } +// if (fieldState.position == 0 && posIncr == 0) { +// throw new IllegalArgumentException("first position increment must be > 0 (got 0)"); +// } +// int position = fieldState.position + posIncr; +// if (position > 0) { +// // NOTE: confusing: this "mirrors" the +// // position++ we do below +// position--; +// } else if (position < 0) { +// throw new IllegalArgumentException("position overflow for field '" + field.name() + "'"); +// } +// +// // position is legal, we can safely place it in fieldState now. +// // not sure if anything will use fieldState after non-aborting +// // exc... +// fieldState.position = position; +// +// if (posIncr == 0) +// fieldState.numOverlap++; +// +// if (checkOffsets) { +// int startOffset = fieldState.offset + offsetAttribute.startOffset(); +// int endOffset = fieldState.offset + offsetAttribute.endOffset(); +// if (startOffset < 0 || endOffset < startOffset) { +// throw new IllegalArgumentException( +// "startOffset must be non-negative, and endOffset must be >= startOffset, " + "startOffset=" +// + startOffset + ",endOffset=" + endOffset); +// } +// if (startOffset < lastStartOffset) { +// throw new IllegalArgumentException("offsets must not go backwards startOffset=" + startOffset +// + " is < lastStartOffset=" + lastStartOffset); +// } +// lastStartOffset = startOffset; +// } +// +// boolean success = false; +// try { +// // If we hit an exception in here, we abort +// // all buffered documents since the last +// // flush, on the likelihood that the +// // internal state of the consumer is now +// // corrupt and should not be flushed to a +// // new segment: +// consumer.add(); +// success = true; +// } finally { +// if (!success) { +// docState.docWriter.setAborting(); +// } +// } +// fieldState.length++; +// fieldState.position++; +// } while (stream.incrementToken()); +// } +// // trigger streams to perform end-of-stream operations +// stream.end(); +// +// fieldState.offset += offsetAttribute.endOffset(); +// success2 = true; +// } finally { +// if (!success2) { +// IOUtils.closeWhileHandlingException(stream); +// } else { +// stream.close(); +// } +// } +// +// fieldState.offset += analyzed ? docState.analyzer.getOffsetGap(fieldInfo.name) : 0; +// fieldState.boost *= field.boost(); +// } +// +// // LUCENE-2387: don't hang onto the field, so GC can +// // reclaim +// fields[i] = null; +// } +// +// consumer.finish(); +// endConsumer.finish(); +// } + } private Configuration _conf; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-shell/src/main/java/org/apache/blur/shell/DescribePlatformCommandCommand.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/DescribePlatformCommandCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/DescribePlatformCommandCommand.java index 5ab7c89..1541010 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/DescribePlatformCommandCommand.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/DescribePlatformCommandCommand.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Map.Entry; import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thrift.generated.ArgumentDescriptor; import org.apache.blur.thrift.generated.Blur; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.CommandDescriptor; @@ -64,25 +65,35 @@ public class DescribePlatformCommandCommand extends Command implements CommandFi out.println(addWhiteSpace("Version:", width) + version); } - Map<String, String> requiredArguments = commandDescriptor.getRequiredArguments(); + Map<String, ArgumentDescriptor> requiredArguments = commandDescriptor.getRequiredArguments(); if (requiredArguments != null && !requiredArguments.isEmpty()) { out.println(); out.println("Required Arguments:"); - for (Entry<String, String> e : requiredArguments.entrySet()) { - out.println(addWhiteSpace("-" + e.getKey(), width) + e.getValue()); + for (Entry<String, ArgumentDescriptor> e : requiredArguments.entrySet()) { + out.println(addWhiteSpace("-" + e.getKey(), width) + toString(e.getValue())); } } - Map<String, String> optionalArguments = commandDescriptor.getOptionalArguments(); + Map<String, ArgumentDescriptor> optionalArguments = commandDescriptor.getOptionalArguments(); if (optionalArguments != null && !optionalArguments.isEmpty()) { out.println(); out.println("Optional Arguments"); - for (Entry<String, String> e : optionalArguments.entrySet()) { - out.println(addWhiteSpace("-" + e.getKey(), width) + e.getValue()); + for (Entry<String, ArgumentDescriptor> e : optionalArguments.entrySet()) { + out.println(addWhiteSpace("-" + e.getKey(), width) + toString(e.getValue())); } } out.println(); } + private String toString(ArgumentDescriptor argumentDescriptor) { + String type = argumentDescriptor.getType(); + String description = argumentDescriptor.getDescription(); + if (description == null) { + return "Type: [" + type + "]"; + } else { + return "Type: [" + type + "] Description: " + description; + } + } + @Override public String description() { return "Describes the specifed command.";
