More fixes based issues found during command creation. The biggest issue is that Lucene field infos are not consistent between indexing and fetching.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/d709ddb8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/d709ddb8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/d709ddb8 Branch: refs/heads/0.2-dev Commit: d709ddb803a9bdd46abbd4a04286fa3b592ba382 Parents: eb5421c Author: Aaron McCurry <[email protected]> Authored: Fri Jan 4 16:24:32 2013 -0500 Committer: Aaron McCurry <[email protected]> Committed: Fri Jan 4 16:24:32 2013 -0500 ---------------------------------------------------------------------- .../java/org/apache/blur/thrift/BlurServer.java | 5 +- .../apache/blur/utils/ThriftLuceneConversion.java | 11 +- .../java/org/apache/blur/shell/AddCommand.java | 157 +++++++++++---- .../java/org/apache/blur/shell/CreateCommand.java | 9 +- .../org/apache/blur/shell/DescribeCommand.java | 2 +- .../org/apache/blur/shell/DisableTableCommand.java | 2 +- .../org/apache/blur/shell/EnableTableCommand.java | 2 +- 7 files changed, 139 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d709ddb8/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java index 3c5ae36..c148d3b 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -69,6 +70,7 @@ import org.apache.blur.utils.BlurConstants; import org.apache.blur.utils.BlurUtil; import org.apache.blur.utils.ThriftLuceneConversion; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.IndexSearcher; import org.apache.thrift.TException; @@ -326,7 +328,8 @@ public class BlurServer extends TableAdmin implements Iface { if (searcher == null) { result.addAll(forwardDoc(session, shardIndex, docLocation, fieldsToLoad)); } else { - result.add(toThrift(searcher.document(docId, fieldsToLoad))); + org.apache.lucene.document.Document document = searcher.document(docId, fieldsToLoad); + result.add(toThrift(document)); } } return result; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d709ddb8/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java b/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java index a71210e..12163ce 100644 --- a/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java +++ b/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java @@ -32,7 +32,6 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.QueryParser; -import org.apache.lucene.queryparser.classic.QueryParserBase; import org.apache.lucene.search.Filter; import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; @@ -115,8 +114,14 @@ public class ThriftLuceneConversion { result.setType(TYPE.BINARY); result.setValue(toByteBuffer(bytes)); } else { - result.setType(TYPE.STRING); - result.setValue(toByteBuffer(field.stringValue())); + //@TODO this is a bit of a hack + if (field.fieldType().omitNorms()) { + result.setType(TYPE.STRING); + result.setValue(toByteBuffer(field.stringValue())); + } else{ + result.setType(TYPE.TEXT); + result.setValue(toByteBuffer(field.stringValue())); + } } } return result; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d709ddb8/src/blur-shell/src/main/java/org/apache/blur/shell/AddCommand.java ---------------------------------------------------------------------- diff --git a/src/blur-shell/src/main/java/org/apache/blur/shell/AddCommand.java b/src/blur-shell/src/main/java/org/apache/blur/shell/AddCommand.java index 71c989a..e960eb7 100644 --- a/src/blur-shell/src/main/java/org/apache/blur/shell/AddCommand.java +++ b/src/blur-shell/src/main/java/org/apache/blur/shell/AddCommand.java @@ -19,59 +19,146 @@ package org.apache.blur.shell; import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; import jline.console.ConsoleReader; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.thrift.generated.Document; +import org.apache.blur.thrift.generated.Field; +import org.apache.blur.thrift.generated.MutateOptions; +import org.apache.blur.thrift.generated.TYPE; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; import org.apache.thrift.TException; public class AddCommand extends Command { + private static final String HELP = "help"; + private static final String WAL = "wal"; + private static final String VISIBLE = "visible"; + private static final String SHARD = "shard"; + public AddCommand(Iface client) { super(client); } + @SuppressWarnings("unchecked") @Override - public void doit(PrintWriter out, String[] args, ConsoleReader reader) throws CommandException, TException, BlurException { - if (args.length != 7) { - throw new CommandException("Invalid args: " + help()); + public void doit(PrintWriter out, String[] args, ConsoleReader reader) throws CommandException, TException, BlurException, ParseException { + Options options = new Options(); + options.addOption("h", HELP, false, "Print the help for this command."); + options.addOption("s", SHARD, true, "Shard to add the documents."); + options.addOption("v", VISIBLE, false, "Wait for the data to be visible."); + options.addOption("w", WAL, false, "Enable write ahead log."); + + CommandLineParser parser = new PosixParser(); + CommandLine line = parser.parse(options, args); + List<String> argList = line.getArgList(); + + if (line.hasOption(HELP) || argList.size() < 3) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp(help(), options); + return; + } + + String tablename = argList.get(1); + + Iface client = getClient(); + MutateOptions moptions = new MutateOptions(); + moptions.setTable(tablename); + if (line.hasOption(SHARD)) { + moptions.setShardIndex(Integer.parseInt(line.getOptionValue(SHARD))); + } + if (line.hasOption(VISIBLE)) { + moptions.setWaitToBeVisible(true); + } else { + moptions.setWaitToBeVisible(false); + } + + if (line.hasOption(WAL)) { + moptions.setWriteAheadLog(true); + } else { + moptions.setWriteAheadLog(false); + } + + Document doc = new Document(); + + for (int i = 2; i < argList.size(); i++) { + Field field = getField(argList.get(i)); + doc.addToFields(field); + } + + List<Document> documents = new ArrayList<Document>(); + documents.add(doc); + client.addDocuments(moptions, documents); + } + + private Field getField(String arg) throws CommandException { + Field field = new Field(); + String name; + TYPE type = TYPE.TEXT; + float boost = 1.0f; + int indexOfEquals = arg.indexOf(':'); + if (indexOfEquals < 0) { + throw new CommandException("No value specified [" + arg + "]"); + } + int startOfType = arg.indexOf('/'); + if (startOfType < 0) { + // no type or boost + name = arg.substring(0, indexOfEquals); + } else { + name = arg.substring(0, startOfType); + + int startOfBoost = arg.indexOf('/', startOfType + 1); + if (startOfBoost < 0) { + // no boost + type = TYPE.valueOf(arg.substring(startOfType + 1, indexOfEquals)); + } else { + type = TYPE.valueOf(arg.substring(startOfType + 1, startOfBoost)); + boost = Float.parseFloat(arg.substring(startOfBoost + 1, indexOfEquals)); + } + } + field.setName(name); + field.setBoost(boost); + field.setType(type); + field.setValue(getValue(type, arg.substring(indexOfEquals + 1))); + return field; + } + + private byte[] getValue(TYPE type, String valueString) throws CommandException { + switch (type) { + case TEXT: + case STRING: + case BINARY: + try { + return valueString.getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + case DOUBLE: + return ByteBuffer.allocate(8).putDouble(Double.parseDouble(valueString)).array(); + case LONG: + return ByteBuffer.allocate(8).putLong(Long.parseLong(valueString)).array(); + case INT: + return ByteBuffer.allocate(4).putInt(Integer.parseInt(valueString)).array(); + case FLOAT: + return ByteBuffer.allocate(4).putFloat(Float.parseFloat(valueString)).array(); + default: + throw new CommandException("Unknown type [" + type + "]"); } - String tablename = args[1]; - String rowid = args[2]; - String recordid = args[3]; - String columnfamily = args[4]; - String columnname = args[5]; - String value = args[6]; - - throw new CommandException("Needs to be implemented"); - - // List<Column> columns = new ArrayList<Column>(); - // columns.add(new Column(columnname, value)); - // - // Record record = new Record(); - // record.setRecordId(recordid); - // record.setFamily(columnfamily); - // record.setColumns(columns); - // - // RecordMutation recordMutation = new RecordMutation(); - // recordMutation.setRecord(record); - // recordMutation.setRecordMutationType(RecordMutationType.REPLACE_ENTIRE_RECORD); - // - // List<RecordMutation> recordMutations = new ArrayList<RecordMutation>(); - // recordMutations.add(recordMutation); - // - // RowMutation mutation = new RowMutation(); - // mutation.setTable(tablename); - // mutation.setRowId(rowid); - // mutation.setRowMutationType(RowMutationType.REPLACE_ROW); - // mutation.setRecordMutations(recordMutations); - // - // client.mutate(mutation); } @Override public String help() { - return "mutate the specified row, args; tablename rowid recordid columnfamily columnname value"; + return "add <tablename> <field[/type[/boost]]:value> ..."; } @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d709ddb8/src/blur-shell/src/main/java/org/apache/blur/shell/CreateCommand.java ---------------------------------------------------------------------- diff --git a/src/blur-shell/src/main/java/org/apache/blur/shell/CreateCommand.java b/src/blur-shell/src/main/java/org/apache/blur/shell/CreateCommand.java index 8c49ae4..4a283b6 100644 --- a/src/blur-shell/src/main/java/org/apache/blur/shell/CreateCommand.java +++ b/src/blur-shell/src/main/java/org/apache/blur/shell/CreateCommand.java @@ -59,14 +59,9 @@ public class CreateCommand extends Command { try { CommandLine line = parser.parse(options, args); List<String> argList = line.getArgList(); - boolean printHelp = false; if (line.hasOption(HELP) || argList.size() != 4) { - printHelp = true; - } - - if (printHelp) { HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp(getName() + " <table name> <table path> <number of shards>", options); + formatter.printHelp(help(), options); return; } @@ -105,7 +100,7 @@ public class CreateCommand extends Command { @Override public String help() { - return "create the named table, args; tableuri tablename shardcount"; + return "create <table name> <table path> <number of shards>"; } @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d709ddb8/src/blur-shell/src/main/java/org/apache/blur/shell/DescribeCommand.java ---------------------------------------------------------------------- diff --git a/src/blur-shell/src/main/java/org/apache/blur/shell/DescribeCommand.java b/src/blur-shell/src/main/java/org/apache/blur/shell/DescribeCommand.java index 9c803d3..15f1ab5 100644 --- a/src/blur-shell/src/main/java/org/apache/blur/shell/DescribeCommand.java +++ b/src/blur-shell/src/main/java/org/apache/blur/shell/DescribeCommand.java @@ -96,7 +96,7 @@ public class DescribeCommand extends TableSingleArgumentCommand { @Override public String help() { - return "describe the named table, args; tablename"; + return "describe <tablename>"; } @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d709ddb8/src/blur-shell/src/main/java/org/apache/blur/shell/DisableTableCommand.java ---------------------------------------------------------------------- diff --git a/src/blur-shell/src/main/java/org/apache/blur/shell/DisableTableCommand.java b/src/blur-shell/src/main/java/org/apache/blur/shell/DisableTableCommand.java index 3b9d58e..32f9b13 100644 --- a/src/blur-shell/src/main/java/org/apache/blur/shell/DisableTableCommand.java +++ b/src/blur-shell/src/main/java/org/apache/blur/shell/DisableTableCommand.java @@ -50,7 +50,7 @@ public class DisableTableCommand extends TableSingleArgumentCommand { @Override public String help() { - return "change status of the named table, args; tablename"; + return "disable <tablename>"; } @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d709ddb8/src/blur-shell/src/main/java/org/apache/blur/shell/EnableTableCommand.java ---------------------------------------------------------------------- diff --git a/src/blur-shell/src/main/java/org/apache/blur/shell/EnableTableCommand.java b/src/blur-shell/src/main/java/org/apache/blur/shell/EnableTableCommand.java index 0664fe4..86da060 100644 --- a/src/blur-shell/src/main/java/org/apache/blur/shell/EnableTableCommand.java +++ b/src/blur-shell/src/main/java/org/apache/blur/shell/EnableTableCommand.java @@ -50,7 +50,7 @@ public class EnableTableCommand extends TableSingleArgumentCommand { @Override public String help() { - return "change status of the named table, args; tablename"; + return "enable <tablename>"; } @Override
