Repository: incubator-blur Updated Branches: refs/heads/apache-blur-0.2 4b62189fb -> 56bf3611d
More updates to query v2. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/c0ea4287 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/c0ea4287 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/c0ea4287 Branch: refs/heads/apache-blur-0.2 Commit: c0ea4287f492d1e42a51810d671f921f0f50f482 Parents: 2200f3a Author: Aaron McCurry <[email protected]> Authored: Mon Jun 23 07:22:59 2014 -0400 Committer: Aaron McCurry <[email protected]> Committed: Mon Jun 23 07:22:59 2014 -0400 ---------------------------------------------------------------------- .../org/apache/blur/shell/QueryCommand2.java | 155 +++++------ .../apache/blur/shell/QueryCommandHelper.java | 263 +++++++++++-------- .../org/apache/blur/shell/TableDisplay.java | 12 +- 3 files changed, 247 insertions(+), 183 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c0ea4287/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand2.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand2.java b/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand2.java index 62e0fdd..86d7438 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand2.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand2.java @@ -30,7 +30,6 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicBoolean; -import jline.Terminal; import jline.console.ConsoleReader; import org.apache.blur.shell.PagingPrintWriter.FinishedException; @@ -44,11 +43,9 @@ import org.apache.blur.thrift.generated.Column; import org.apache.blur.thrift.generated.ErrorType; import org.apache.blur.thrift.generated.FetchResult; import org.apache.blur.thrift.generated.FetchRowResult; -import org.apache.blur.thrift.generated.HighlightOptions; import org.apache.blur.thrift.generated.Record; import org.apache.blur.thrift.generated.Row; -import org.apache.blur.thrift.generated.Query; -import org.apache.blur.thrift.generated.Selector; +import org.apache.commons.cli.CommandLine; public class QueryCommand2 extends Command implements TableFirstArgCommand { @Override @@ -57,10 +54,9 @@ public class QueryCommand2 extends Command implements TableFirstArgCommand { if (args.length < 3) { throw new CommandException("Invalid args: " + help()); } - PagingPrintWriter out = new PagingPrintWriter(outPw); try { - doItInternal(client, args, out); + doItInternal(client, args, outPw); } catch (FinishedException e) { if (Main.debug) { e.printStackTrace(); @@ -68,37 +64,21 @@ public class QueryCommand2 extends Command implements TableFirstArgCommand { } } - private void doItInternal(Blur.Iface client, String[] args, PagingPrintWriter out) throws FinishedException, - BlurException, TException { + private void doItInternal(Blur.Iface client, String[] args, PrintWriter out) throws FinishedException, BlurException, + TException { String tablename = args[1]; - String queryStr = ""; - for (int i = 2; i < args.length; i++) { - queryStr += args[i] + " "; - } - - BlurQuery blurQuery = new BlurQuery(); - Query query = new Query(); - query.setQuery(queryStr); - blurQuery.setQuery(query); - blurQuery.setSelector(new Selector(Main.selector)); - blurQuery.setCacheResult(false); - blurQuery.setUseCacheIfPresent(false); - - if (Main.highlight) { - blurQuery.getSelector().setHighlightOptions(new HighlightOptions()); - } - + CommandLine commandLine = QueryCommandHelper.parse(args, out); + BlurQuery blurQuery = QueryCommandHelper.getBlurQuery(commandLine); if (Main.debug) { out.println(blurQuery); } - ConsoleReader reader = getConsoleReader(); if (reader == null) { throw new BlurException("This command can only be run with a proper jline environment.", null, ErrorType.UNKNOWN); } String prompt = reader.getPrompt(); reader.setPrompt(""); - TableDisplay tableDisplay = new TableDisplay(reader); + final TableDisplay tableDisplay = new TableDisplay(reader); tableDisplay.setSeperator(" "); try { @@ -115,55 +95,13 @@ public class QueryCommand2 extends Command implements TableFirstArgCommand { synchronized (viewing) { viewing.set(false); viewing.notify(); + tableDisplay.setStopReadingInput(true); } } }, 'q'); - int line = 0; - - tableDisplay.setHeader(0, "rowid"); - tableDisplay.setHeader(1, "family"); - tableDisplay.setHeader(2, "recordid"); - - Map<String, List<String>> columnOrder = new HashMap<String, List<String>>(); - - for (BlurResult result : blurResults.getResults()) { - FetchResult fetchResult = result.getFetchResult(); - FetchRowResult rowResult = fetchResult.getRowResult(); - if (rowResult != null) { - Row row = rowResult.getRow(); - String id = row.getId(); - tableDisplay.set(0, line, id); - List<Record> records = order(row.getRecords()); - String currentFamily = "#"; - for (Record record : records) { - int c = 3; - List<String> orderedColumns = getOrderColumnValues(record, columnOrder); - String family = record.getFamily(); - if (!family.equals(currentFamily)) { - tableDisplay.set(1, line, family); - List<String> list = columnOrder.get(family); - for (int i = 0; i < list.size(); i++) { - tableDisplay.set(i + c, line, highlight(list.get(i))); - } - currentFamily = family; - line++; - } - - tableDisplay.set(2, line, record.getRecordId()); + renderRowMultiFamily(tableDisplay, blurResults); - for (String oc : orderedColumns) { - if (oc != null) { - tableDisplay.set(c, line, oc); - } - c++; - } - line++; - } - } else { - throw new RuntimeException("impl"); - } - } while (viewing.get()) { synchronized (viewing) { try { @@ -174,17 +112,84 @@ public class QueryCommand2 extends Command implements TableFirstArgCommand { } } } finally { - reader.setPrompt(prompt); try { tableDisplay.close(); - } catch (IOException ex) { - ex.printStackTrace(); + } catch (IOException e) { + if (Main.debug) { + e.printStackTrace(); + } + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + if (Main.debug) { + e.printStackTrace(); + } + } + try { + reader.setPrompt(""); + reader.clearScreen(); + } catch (IOException e) { + if (Main.debug) { + e.printStackTrace(); + } + } + out.write("\u001B[0m"); + out.flush(); + reader.setPrompt(prompt); + } + } + + private void renderRowMultiFamily(TableDisplay tableDisplay, BlurResults blurResults) { + int line = 0; + + tableDisplay.setHeader(0, white("rowid")); + tableDisplay.setHeader(1, white("recordid")); + + Map<String, List<String>> columnOrder = new HashMap<String, List<String>>(); + + for (BlurResult result : blurResults.getResults()) { + FetchResult fetchResult = result.getFetchResult(); + FetchRowResult rowResult = fetchResult.getRowResult(); + if (rowResult != null) { + Row row = rowResult.getRow(); + String id = row.getId(); + tableDisplay.set(0, line, white(id)); + List<Record> records = order(row.getRecords()); + String currentFamily = "#"; + for (Record record : records) { + int c = 2; + List<String> orderedColumns = getOrderColumnValues(record, columnOrder); + String family = record.getFamily(); + tableDisplay.set(1, line, white(record.getRecordId())); + if (!family.equals(currentFamily)) { + List<String> list = columnOrder.get(family); + for (int i = 0; i < list.size(); i++) { + tableDisplay.set(i + c, line, highlight(family + "." + list.get(i))); + } + currentFamily = family; + line++; + } + for (String oc : orderedColumns) { + if (oc != null) { + tableDisplay.set(c, line, white(oc)); + } + c++; + } + line++; + } + } else { + throw new RuntimeException("impl"); } } } + private String white(String s) { + return "\u001B[0m" + s; + } + private String highlight(String s) { - return "\u001B[33m" + s + "\u001B[0m"; + return "\u001B[33m" + s; } private List<Record> order(List<Record> records) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c0ea4287/blur-shell/src/main/java/org/apache/blur/shell/QueryCommandHelper.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/QueryCommandHelper.java b/blur-shell/src/main/java/org/apache/blur/shell/QueryCommandHelper.java index e9baf2c..576b162 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/QueryCommandHelper.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/QueryCommandHelper.java @@ -20,6 +20,7 @@ package org.apache.blur.shell; import java.io.PrintWriter; import java.io.Writer; +import java.util.ArrayList; import java.util.List; import org.apache.blur.thrift.generated.BlurQuery; @@ -32,155 +33,203 @@ import org.apache.blur.thrift.generated.SortField; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.cli.PosixParser; public class QueryCommandHelper { - @SuppressWarnings("static-access") - public static CommandLine parse(String[] otherArgs, Writer out) { - Options options = new Options(); - options.addOption( - OptionBuilder - .withDescription("Disables row query. (Enabled by default)") - .create("R")); - options.addOption( - OptionBuilder - .withDescription("Disables row query. (Enabled by default)") - .create("R")); - - String queryStr; - boolean rowQuery; - ScoreType scoreType; - String recordFilter; - String rowFilter; - String rowId; - long start; - int fetch; - long maxQueryTime; - long minimumNumberOfResults; - List<Facet> facets; - List<SortField> sortFields; - - + + private static final String SORT = "sort"; + private static final String FACET = "facet"; + private static final String ROW_ID = "rowId"; + private static final String MINIMUM_NUMBER_OF_RESULTS = "minimumNumberOfResults"; + private static final String MAX_QUERY_TIME = "maxQueryTime"; + private static final String FETCH = "fetch"; + private static final String START = "start"; + private static final String DISABLE_ROW_QUERY = "disableRowQuery"; + private static final String SCORE_TYPE = "scoreType"; + private static final String RECORD_FILTER = "recordFilter"; + private static final String ROW_FILTER = "rowFilter"; + + @SuppressWarnings("unchecked") + public static BlurQuery getBlurQuery(CommandLine commandLine) { + List<String> argList = commandLine.getArgList(); + Option[] options = commandLine.getOptions(); + Query query = new Query(); - query.setQuery(queryStr); - query.setRecordFilter(recordFilter); - query.setRowFilter(rowFilter); - query.setRowQuery(rowQuery); - query.setScoreType(scoreType); - + // Start at 2 because 1st arg is command 2nd is table + query.setQuery(join(argList.subList(2, argList.size()), " ")); + if (commandLine.hasOption(DISABLE_ROW_QUERY)) { + query.setRowQuery(false); + } + if (commandLine.hasOption(SCORE_TYPE)) { + String scoreTypeStr = commandLine.getOptionValue(SCORE_TYPE); + ScoreType scoreType = ScoreType.valueOf(scoreTypeStr.toUpperCase()); + query.setScoreType(scoreType); + } + if (commandLine.hasOption(RECORD_FILTER)) { + String recordFilter = commandLine.getOptionValue(RECORD_FILTER); + query.setRecordFilter(recordFilter); + } + if (commandLine.hasOption(ROW_FILTER)) { + String rowFilter = commandLine.getOptionValue(ROW_FILTER); + query.setRecordFilter(rowFilter); + } + + // String recordFilter; + // String rowFilter; + // String rowId; + // long start; + // int fetch; + // long maxQueryTime; + // long minimumNumberOfResults; + // List<Facet> facets; + // List<SortField> sortFields; + BlurQuery blurQuery = new BlurQuery(); blurQuery.setQuery(query); blurQuery.setSelector(new Selector(Main.selector)); blurQuery.setCacheResult(false); blurQuery.setUseCacheIfPresent(false); - blurQuery.setFacets(facets); - blurQuery.setFetch(fetch); - blurQuery.setMaxQueryTime(maxQueryTime); - blurQuery.setMinimumNumberOfResults(minimumNumberOfResults); - blurQuery.setRowId(rowId); - blurQuery.setSortFields(sortFields); - blurQuery.setStart(start); + + if (commandLine.hasOption(START)) { + String startStr = commandLine.getOptionValue(START); + blurQuery.setStart(Long.parseLong(startStr)); + } + if (commandLine.hasOption(FETCH)) { + String fetchStr = commandLine.getOptionValue(FETCH); + blurQuery.setFetch(Integer.parseInt(fetchStr)); + } + if (commandLine.hasOption(MAX_QUERY_TIME)) { + String maxQueryTimeStr = commandLine.getOptionValue(MAX_QUERY_TIME); + blurQuery.setMaxQueryTime(Long.parseLong(maxQueryTimeStr)); + } + if (commandLine.hasOption(MINIMUM_NUMBER_OF_RESULTS)) { + String minNumbResultsStr = commandLine.getOptionValue(MINIMUM_NUMBER_OF_RESULTS); + blurQuery.setMinimumNumberOfResults(Long.parseLong(minNumbResultsStr)); + } + if (commandLine.hasOption(ROW_ID)) { + String rowId = commandLine.getOptionValue(ROW_FILTER); + blurQuery.setRowId(rowId); + } + List<Facet> facets = new ArrayList<Facet>(); + for (Option option : options) { + if (option.getOpt().equals(FACET)) { + List<String> valuesList = option.getValuesList(); + Facet facet = new Facet(); + facet.setQueryStr(join(valuesList, " ")); + facets.add(facet); + } + } + if (!facets.isEmpty()) { + blurQuery.setFacets(facets); + } + + List<SortField> sortFields = new ArrayList<SortField>(); + for (Option option : options) { + if (option.getOpt().equals(SORT)) { + List<String> valuesList = option.getValuesList(); + if (valuesList.size() == 2) { + sortFields.add(new SortField(valuesList.get(0), valuesList.get(1), false)); + } else if (valuesList.size() == 3) { + sortFields.add(new SortField(valuesList.get(0), valuesList.get(1), Boolean.parseBoolean(valuesList.get(2)))); + } else { + throw new RuntimeException("Sort take 2 or 3 parameters."); + } + } + } + if (!sortFields.isEmpty()) { + blurQuery.setSortFields(sortFields); + } if (Main.highlight) { blurQuery.getSelector().setHighlightOptions(new HighlightOptions()); } - - - + return blurQuery; + } + + private static String join(List<String> argList, String sep) { + StringBuilder builder = new StringBuilder(); + for (String s : argList) { + if (builder.length() != 0) { + builder.append(sep); + } + builder.append(s); + } + return builder.toString(); + } + + @SuppressWarnings("static-access") + public static CommandLine parse(String[] otherArgs, Writer out) { + Options options = new Options(); options.addOption( OptionBuilder - .withDescription("Disables the table when it is created. (Enabled by default)") - .create("d")); - + .withDescription("Disables row query. (Enabled by default)") + .create(DISABLE_ROW_QUERY)); options.addOption( OptionBuilder - .withDescription("Enabled strict types on a table. (Disabled by default)") - .create("s")); - + .hasArg() + .withArgName(SCORE_TYPE) + .withDescription("Specify the scoring type.") + .create(SCORE_TYPE)); options.addOption( OptionBuilder - .withDescription("Enables a read only table. (Disabled by default)") - .create("r")); - + .hasArgs() + .withArgName(ROW_FILTER) + .withDescription("Specify row filter.") + .create(ROW_FILTER)); options.addOption( OptionBuilder - .isRequired() - .hasArg() - .withArgName("tablename") - .withDescription("* The table name.") - .create("t")); - + .hasArgs() + .withArgName(RECORD_FILTER) + .withDescription("Specify record filter.") + .create(RECORD_FILTER)); options.addOption( OptionBuilder - .isRequired() .hasArg() - .withArgName("shard count") - .withDescription("* The number of shards in the table.") - .create("c")); - + .withArgName(START) + .withDescription("Specify the starting position (paging).") + .create(START)); options.addOption( OptionBuilder .hasArg() - .withArgName("uri") - .withDescription("The location of the table. (Example hdfs://namenode/blur/tables/table)") - .create("l")); - + .withArgName(FETCH) + .withDescription("Specify the number of elements to fetch in a single page.") + .create(FETCH)); options.addOption( OptionBuilder - .withArgName("filetype") - .hasOptionalArgs() - .withDescription("Sets the filetypes (.tim, .tis, .doc, etc.) to be cached in the block cache. (All by default)") - .create("B")); - - options.addOption( - OptionBuilder - .withDescription("If table is not strict, disables the missing field, fieldless indexing. (Enabled by default)") - .create("mfi")); - - options.addOption( - OptionBuilder - .withArgName("field type") .hasArg() - .withDescription("If table is not strict, sets the field type for the missing field. (text by default)") - .create("mft")); - + .withArgName(MAX_QUERY_TIME) + .withDescription("Specify the maximum amount of time to allow query to execute.") + .create(MAX_QUERY_TIME)); options.addOption( OptionBuilder - .withArgName("name value") - .hasArgs(2) - .withDescription("If table is not strict, sets the properties for the missing field.") - .create("mfp")); - + .hasArg() + .withArgName(MINIMUM_NUMBER_OF_RESULTS) + .withDescription("Specify the minimum number of results required before returning from query.") + .create(MINIMUM_NUMBER_OF_RESULTS)); options.addOption( OptionBuilder - .withArgName("name value") - .hasArgs(2) - .withDescription("Sets the properties for this table descriptor.") - .create("p")); - + .hasArg() + .withArgName(ROW_ID) + .withDescription("Specify the rowId to execute the query against (this reduces the spray to other shards).") + .create(ROW_ID)); options.addOption( OptionBuilder - .withArgName("column name*") + .withArgName(FACET) .hasArgs() - .withDescription("Sets what columns to pre cache during warmup. (By default all columns are cached)") - .create("P")); - + .withDescription("Specify facet to be executed with this query.") + .create(FACET)); options.addOption( OptionBuilder - .withArgName("classname") - .hasArg() - .withDescription("Sets the similarity class for the table. (By org.apache.blur.lucene.search.FairSimilarity is used)") - .create("S")); + .withArgName(SORT) + .hasArgs() + .withDescription("Specify a sort to be applied to this query <family> <column> [<reverse>].") + .create(SORT)); - options.addOption( - OptionBuilder - .withDescription("Displays help for this command.") - .create("h")); - CommandLineParser parser = new PosixParser(); CommandLine cmd = null; try { @@ -188,14 +237,14 @@ public class QueryCommandHelper { if (cmd.hasOption("h")) { HelpFormatter formatter = new HelpFormatter(); PrintWriter pw = new PrintWriter(out, true); - formatter.printHelp(pw, HelpFormatter.DEFAULT_WIDTH, "create", null, options, + formatter.printHelp(pw, HelpFormatter.DEFAULT_WIDTH, "query", null, options, HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD, null, false); return null; } } catch (ParseException e) { HelpFormatter formatter = new HelpFormatter(); PrintWriter pw = new PrintWriter(out, true); - formatter.printHelp(pw, HelpFormatter.DEFAULT_WIDTH, "create", null, options, + formatter.printHelp(pw, HelpFormatter.DEFAULT_WIDTH, "query", null, options, HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD, null, false); return null; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/c0ea4287/blur-shell/src/main/java/org/apache/blur/shell/TableDisplay.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/TableDisplay.java b/blur-shell/src/main/java/org/apache/blur/shell/TableDisplay.java index c05533d..0cf31cf 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/TableDisplay.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/TableDisplay.java @@ -125,6 +125,7 @@ public class TableDisplay implements Closeable { private String _seperator = SEP; private final Thread _inputReaderThread; private final Map<Integer, Runnable> _keyHookMap = new ConcurrentHashMap<Integer, Runnable>(); + private final AtomicBoolean _stopReadingInput = new AtomicBoolean(false); public void setSeperator(String seperator) { _seperator = seperator; @@ -161,7 +162,7 @@ public class TableDisplay implements Closeable { try { InputStream input = _reader.getInput(); int read; - while (_running.get() && (read = input.read()) != -1) { + while (!_stopReadingInput.get() && _running.get() && (read = input.read()) != -1) { if (read == 27) { if (input.read() == 91) { read = input.read(); @@ -352,6 +353,11 @@ public class TableDisplay implements Closeable { if (_running.get()) { _running.set(false); _inputReaderThread.interrupt(); + try { + _inputReaderThread.join(); + } catch (InterruptedException e) { + throw new IOException(e); + } } } @@ -585,4 +591,8 @@ public class TableDisplay implements Closeable { } } + + public void setStopReadingInput(boolean b) { + _stopReadingInput.set(true); + } }
