Repository: incubator-blur Updated Branches: refs/heads/apache-blur-0.2 23e3892cf -> 2200f3a57
Working on v2 of the query command. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/b63ca2c0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/b63ca2c0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/b63ca2c0 Branch: refs/heads/apache-blur-0.2 Commit: b63ca2c05626aeede2cbcf769f3394790bed2cd1 Parents: 198cca0 Author: Aaron McCurry <[email protected]> Authored: Sun Jun 22 13:04:11 2014 -0400 Committer: Aaron McCurry <[email protected]> Committed: Sun Jun 22 13:04:11 2014 -0400 ---------------------------------------------------------------------- .../org/apache/blur/shell/QueryCommand2.java | 121 ++++++++++- .../apache/blur/shell/QueryCommandHelper.java | 204 ++++++++++++++++++ .../org/apache/blur/shell/TableDisplay.java | 207 ++++++++++++++++--- 3 files changed, 499 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b63ca2c0/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand2.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand2.java b/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand2.java index bce3a16..62e0fdd 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand2.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand2.java @@ -20,7 +20,14 @@ package org.apache.blur.shell; import java.io.IOException; import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.atomic.AtomicBoolean; import jline.Terminal; @@ -118,20 +125,37 @@ public class QueryCommand2 extends Command implements TableFirstArgCommand { tableDisplay.setHeader(1, "family"); tableDisplay.setHeader(2, "recordid"); + Map<String, List<String>> columnOrder = new HashMap<String, List<String>>(); + for (BlurResult result : blurResults.getResults()) { FetchResult fetchResult = result.getFetchResult(); FetchRowResult rowResult = fetchResult.getRowResult(); if (rowResult != null) { Row row = rowResult.getRow(); String id = row.getId(); - List<Record> records = row.getRecords(); + tableDisplay.set(0, line, id); + List<Record> records = order(row.getRecords()); + String currentFamily = "#"; for (Record record : records) { - tableDisplay.set(0, line, id); - tableDisplay.set(1, line, record.getFamily()); - tableDisplay.set(2, line, record.getRecordId()); int c = 3; - for (Column column : record.getColumns()) { - tableDisplay.set(c, line, column.getName() + " " + column.getValue()); + List<String> orderedColumns = getOrderColumnValues(record, columnOrder); + String family = record.getFamily(); + if (!family.equals(currentFamily)) { + tableDisplay.set(1, line, family); + List<String> list = columnOrder.get(family); + for (int i = 0; i < list.size(); i++) { + tableDisplay.set(i + c, line, highlight(list.get(i))); + } + currentFamily = family; + line++; + } + + tableDisplay.set(2, line, record.getRecordId()); + + for (String oc : orderedColumns) { + if (oc != null) { + tableDisplay.set(c, line, oc); + } c++; } line++; @@ -159,6 +183,91 @@ public class QueryCommand2 extends Command implements TableFirstArgCommand { } } + private String highlight(String s) { + return "\u001B[33m" + s + "\u001B[0m"; + } + + private List<Record> order(List<Record> records) { + List<Record> list = new ArrayList<Record>(records); + Collections.sort(list, new Comparator<Record>() { + @Override + public int compare(Record o1, Record o2) { + String family1 = o1.getFamily(); + String family2 = o2.getFamily(); + String recordId1 = o1.getRecordId(); + String recordId2 = o2.getRecordId(); + if (family1 == null && family2 == null) { + return recordId1.compareTo(recordId2); + } + if (family1 == null) { + return -1; + } + int compareTo = family1.compareTo(family2); + if (compareTo == 0) { + return recordId1.compareTo(recordId2); + } + return compareTo; + } + }); + return list; + } + + private List<String> getOrderColumnValues(Record record, Map<String, List<String>> columnOrder) { + String family = record.getFamily(); + List<String> columnNameList = columnOrder.get(family); + if (columnNameList == null) { + columnOrder.put(family, columnNameList = new ArrayList<String>()); + } + Map<String, List<Column>> columnMap = getColumnMap(record); + Set<String> recordColumnNames = new TreeSet<String>(columnMap.keySet()); + for (String cn : recordColumnNames) { + if (!columnNameList.contains(cn)) { + columnNameList.add(cn); + } + } + List<String> result = new ArrayList<String>(); + for (String col : columnNameList) { + List<Column> list = columnMap.get(col); + if (list != null) { + result.add(toString(list)); + } else { + result.add(null); + } + } + return result; + } + + private String toString(List<Column> list) { + if (list.size() == 0) { + throw new RuntimeException("Should not happen"); + } + if (list.size() == 1) { + return list.get(0).getValue(); + } + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < list.size(); i++) { + Column column = list.get(i); + if (i != 0) { + builder.append(","); + } + builder.append("[").append(column.getValue()).append("]"); + } + return builder.toString(); + } + + private Map<String, List<Column>> getColumnMap(Record record) { + Map<String, List<Column>> map = new HashMap<String, List<Column>>(); + for (Column column : record.getColumns()) { + String name = column.getName(); + List<Column> list = map.get(name); + if (list == null) { + map.put(name, list = new ArrayList<Column>()); + } + list.add(column); + } + return map; + } + private void lineBreak(PagingPrintWriter out, int maxWidth) throws FinishedException { for (int i = 0; i < maxWidth; i++) { out.print('-'); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b63ca2c0/blur-shell/src/main/java/org/apache/blur/shell/QueryCommandHelper.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/QueryCommandHelper.java b/blur-shell/src/main/java/org/apache/blur/shell/QueryCommandHelper.java new file mode 100644 index 0000000..e9baf2c --- /dev/null +++ b/blur-shell/src/main/java/org/apache/blur/shell/QueryCommandHelper.java @@ -0,0 +1,204 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.blur.shell; + +import java.io.PrintWriter; +import java.io.Writer; +import java.util.List; + +import org.apache.blur.thrift.generated.BlurQuery; +import org.apache.blur.thrift.generated.Facet; +import org.apache.blur.thrift.generated.HighlightOptions; +import org.apache.blur.thrift.generated.Query; +import org.apache.blur.thrift.generated.ScoreType; +import org.apache.blur.thrift.generated.Selector; +import org.apache.blur.thrift.generated.SortField; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; + +public class QueryCommandHelper { + @SuppressWarnings("static-access") + public static CommandLine parse(String[] otherArgs, Writer out) { + Options options = new Options(); + options.addOption( + OptionBuilder + .withDescription("Disables row query. (Enabled by default)") + .create("R")); + options.addOption( + OptionBuilder + .withDescription("Disables row query. (Enabled by default)") + .create("R")); + + String queryStr; + boolean rowQuery; + ScoreType scoreType; + String recordFilter; + String rowFilter; + String rowId; + long start; + int fetch; + long maxQueryTime; + long minimumNumberOfResults; + List<Facet> facets; + List<SortField> sortFields; + + + Query query = new Query(); + query.setQuery(queryStr); + query.setRecordFilter(recordFilter); + query.setRowFilter(rowFilter); + query.setRowQuery(rowQuery); + query.setScoreType(scoreType); + + BlurQuery blurQuery = new BlurQuery(); + blurQuery.setQuery(query); + blurQuery.setSelector(new Selector(Main.selector)); + blurQuery.setCacheResult(false); + blurQuery.setUseCacheIfPresent(false); + blurQuery.setFacets(facets); + blurQuery.setFetch(fetch); + blurQuery.setMaxQueryTime(maxQueryTime); + blurQuery.setMinimumNumberOfResults(minimumNumberOfResults); + blurQuery.setRowId(rowId); + blurQuery.setSortFields(sortFields); + blurQuery.setStart(start); + + if (Main.highlight) { + blurQuery.getSelector().setHighlightOptions(new HighlightOptions()); + } + + + + + options.addOption( + OptionBuilder + .withDescription("Disables the table when it is created. (Enabled by default)") + .create("d")); + + options.addOption( + OptionBuilder + .withDescription("Enabled strict types on a table. (Disabled by default)") + .create("s")); + + options.addOption( + OptionBuilder + .withDescription("Enables a read only table. (Disabled by default)") + .create("r")); + + options.addOption( + OptionBuilder + .isRequired() + .hasArg() + .withArgName("tablename") + .withDescription("* The table name.") + .create("t")); + + options.addOption( + OptionBuilder + .isRequired() + .hasArg() + .withArgName("shard count") + .withDescription("* The number of shards in the table.") + .create("c")); + + options.addOption( + OptionBuilder + .hasArg() + .withArgName("uri") + .withDescription("The location of the table. (Example hdfs://namenode/blur/tables/table)") + .create("l")); + + options.addOption( + OptionBuilder + .withArgName("filetype") + .hasOptionalArgs() + .withDescription("Sets the filetypes (.tim, .tis, .doc, etc.) to be cached in the block cache. (All by default)") + .create("B")); + + options.addOption( + OptionBuilder + .withDescription("If table is not strict, disables the missing field, fieldless indexing. (Enabled by default)") + .create("mfi")); + + options.addOption( + OptionBuilder + .withArgName("field type") + .hasArg() + .withDescription("If table is not strict, sets the field type for the missing field. (text by default)") + .create("mft")); + + options.addOption( + OptionBuilder + .withArgName("name value") + .hasArgs(2) + .withDescription("If table is not strict, sets the properties for the missing field.") + .create("mfp")); + + options.addOption( + OptionBuilder + .withArgName("name value") + .hasArgs(2) + .withDescription("Sets the properties for this table descriptor.") + .create("p")); + + options.addOption( + OptionBuilder + .withArgName("column name*") + .hasArgs() + .withDescription("Sets what columns to pre cache during warmup. (By default all columns are cached)") + .create("P")); + + options.addOption( + OptionBuilder + .withArgName("classname") + .hasArg() + .withDescription("Sets the similarity class for the table. (By org.apache.blur.lucene.search.FairSimilarity is used)") + .create("S")); + + options.addOption( + OptionBuilder + .withDescription("Displays help for this command.") + .create("h")); + + CommandLineParser parser = new PosixParser(); + CommandLine cmd = null; + try { + cmd = parser.parse(options, otherArgs); + if (cmd.hasOption("h")) { + HelpFormatter formatter = new HelpFormatter(); + PrintWriter pw = new PrintWriter(out, true); + formatter.printHelp(pw, HelpFormatter.DEFAULT_WIDTH, "create", null, options, + HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD, null, false); + return null; + } + } catch (ParseException e) { + HelpFormatter formatter = new HelpFormatter(); + PrintWriter pw = new PrintWriter(out, true); + formatter.printHelp(pw, HelpFormatter.DEFAULT_WIDTH, "create", null, options, + HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD, null, false); + return null; + } + return cmd; + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b63ca2c0/blur-shell/src/main/java/org/apache/blur/shell/TableDisplay.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/TableDisplay.java b/blur-shell/src/main/java/org/apache/blur/shell/TableDisplay.java index 77013a6..c05533d 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/TableDisplay.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/TableDisplay.java @@ -20,9 +20,10 @@ import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.io.Writer; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; import java.util.Map; -import java.util.Timer; -import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -36,6 +37,11 @@ public class TableDisplay implements Closeable { // System.out.println("\u001B[1mfoo\u001B[0m@bar\u001B[32m@baz\u001B[0m>"); + // \u001B[0m normal + // \u001B[33m yellow + + // System.out.println("\u001B[33mCOLUMN\u001B[0mnormal"); + // // for (int i = 0; i < 100; i++) { // System.out.println("\u001B[0m" + i + "\u001B[" + i + "mfoo"); // } @@ -43,7 +49,7 @@ public class TableDisplay implements Closeable { ConsoleReader reader = new ConsoleReader(); TableDisplay tableDisplay = new TableDisplay(reader); tableDisplay.setSeperator("|"); -// Random random = new Random(); + // Random random = new Random(); int maxX = 20; int maxY = 100; tableDisplay.setHeader(0, ""); @@ -80,7 +86,11 @@ public class TableDisplay implements Closeable { // } for (int x = 0; x < maxX; x++) { for (int y = 0; y < maxY; y++) { - tableDisplay.set(x, y, x + "," + y); + if (x == 7 && y == 7) { + tableDisplay.set(x, y, highlight(x + "," + y)); + } else { + tableDisplay.set(x, y, x + "," + y); + } } } while (running.get()) { @@ -93,6 +103,11 @@ public class TableDisplay implements Closeable { } } + private static String highlight(String s) { + // return s; + return "\u001B[33m" + s + "\u001B[0m"; + } + public void addKeyHook(Runnable runnable, int c) { _keyHookMap.put(c, runnable); } @@ -103,7 +118,7 @@ public class TableDisplay implements Closeable { private final ConcurrentMap<Integer, String> _header = new ConcurrentHashMap<Integer, String>(); private final ConcurrentMap<Integer, Integer> _maxWidth = new ConcurrentHashMap<Integer, Integer>(); private final AtomicBoolean _running = new AtomicBoolean(true); - private final Timer _timer; + private final Thread _paintThread; private final Canvas _canvas; private int _maxY; private int _maxX; @@ -117,25 +132,36 @@ public class TableDisplay implements Closeable { public TableDisplay(ConsoleReader reader) { _reader = reader; - _timer = new Timer("Render Thread", true); _canvas = new Canvas(_reader); - _timer.schedule(new TimerTask() { + _paintThread = new Thread(new Runnable() { @Override public void run() { - try { - render(_canvas); - } catch (IOException e) { - e.printStackTrace(); + while (_running.get()) { + synchronized (_canvas) { + try { + render(_canvas); + } catch (IOException e) { + e.printStackTrace(); + } + try { + _canvas.wait(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } } } - }, 100, 250); + }); + _paintThread.setDaemon(true); + _paintThread.setName("Render Thread"); + _paintThread.start(); _inputReaderThread = new Thread(new Runnable() { @Override public void run() { try { InputStream input = _reader.getInput(); int read; - while ((read = input.read()) != -1 && _running.get()) { + while (_running.get() && (read = input.read()) != -1) { if (read == 27) { if (input.read() == 91) { read = input.read(); @@ -166,6 +192,7 @@ public class TableDisplay implements Closeable { runnable.run(); } } + paint(); } } catch (IOException e) { e.printStackTrace(); @@ -223,7 +250,7 @@ public class TableDisplay implements Closeable { private void buffer(Canvas canvas, String value, int width) { canvas.append(value); - width -= value.length(); + width -= getVisibleLength(value); while (width > 0) { canvas.append(' '); width--; @@ -233,6 +260,7 @@ public class TableDisplay implements Closeable { public void setHeader(int x, String name) { _header.put(x, name); setMaxWidth(x, name); + paint(); } public void set(int x, int y, Object value) { @@ -244,12 +272,19 @@ public class TableDisplay implements Closeable { setMaxX(x); setMaxY(y); setMaxWidth(x, value); + paint(); + } + + private void paint() { + synchronized (_canvas) { + _canvas.notify(); + } } private void setMaxWidth(int x, Object value) { if (value != null) { String s = getString(value); - int length = s.length(); + int length = getVisibleLength(s); while (true) { Integer width = _maxWidth.get(x); if (width == null) { @@ -269,6 +304,26 @@ public class TableDisplay implements Closeable { } } + private int getVisibleLength(String s) { + int length = 0; + boolean color = false; + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (color) { + if (c == 'm') { + color = false; + } + } else { + if (c == 27) { + color = true; + } else { + length++; + } + } + } + return length; + } + private String getString(Object value) { if (value != null) { return value.toString(); @@ -296,8 +351,6 @@ public class TableDisplay implements Closeable { public void close() throws IOException { if (_running.get()) { _running.set(false); - _timer.cancel(); - _timer.purge(); _inputReaderThread.interrupt(); } } @@ -354,10 +407,107 @@ public class TableDisplay implements Closeable { } } + public static class LineBuilder { + + static class LineBuilderPart { + String _visible; + String _notVisible; + } + + private List<LineBuilderPart> _parts = new ArrayList<LineBuilderPart>(); + private boolean _readingColor; + private StringBuilder _currentNotVisible = new StringBuilder(); + private StringBuilder _currentVisible = new StringBuilder(); + + public void reset() { + _currentNotVisible.setLength(0); + _currentVisible.setLength(0); + _parts.clear(); + } + + public void close() { + startNewPart(); + } + + public int visibleLength() { + int l = 0; + for (LineBuilderPart p : _parts) { + l += p._visible.length(); + } + return l; + } + + public String substring(int start, int end) { + StringBuilder b = new StringBuilder(); + Iterator<LineBuilderPart> iterator = _parts.iterator(); + LineBuilderPart current = iterator.next(); + int partPos = 0; + boolean currentPartHasEmittedNonVisible = false; + for (int i = 0; i < end; i++) { + if (partPos >= current._visible.length()) { + current = iterator.next(); + partPos = 0; + currentPartHasEmittedNonVisible = false; + } + if (isVisible(i, start, end)) { + if (!currentPartHasEmittedNonVisible) { + b.append(current._notVisible); + currentPartHasEmittedNonVisible = true; + } + b.append(current._visible.charAt(partPos)); + } + partPos++; + } + return b.toString(); + } + + private boolean isVisible(int i, int start, int end) { + if (i >= start && i < end) { + return true; + } + return false; + } + + public void append(char c) { + if (_readingColor) { + if (c == 'm') { + _readingColor = false; + _currentNotVisible.append(c); + } else { + _currentNotVisible.append(c); + } + } else { + if (c == 27) { + startNewPart(); + _readingColor = true; + _currentNotVisible.append(c); + } else { + _currentVisible.append(c); + } + } + } + + private void startNewPart() { + LineBuilderPart part = new LineBuilderPart(); + part._notVisible = _currentNotVisible.toString(); + part._visible = _currentVisible.toString(); + _parts.add(part); + _currentNotVisible.setLength(0); + _currentVisible.setLength(0); + } + + public void append(String s) { + for (int i = 0; i < s.length(); i++) { + append(s.charAt(i)); + } + } + + } + static class Canvas { - private final StringBuilder _builder = new StringBuilder(); - private final StringBuilder _lineBuilder = new StringBuilder(); + private final StringBuilder _screenBuilder = new StringBuilder(); + private final LineBuilder _lineBuilder = new LineBuilder(); private final ConsoleReader _reader; private int _height; private int _width; @@ -372,7 +522,7 @@ public class TableDisplay implements Closeable { public void reset() { _line = 0; - _builder.setLength(0); + _screenBuilder.setLength(0); Terminal terminal = _reader.getTerminal(); _height = terminal.getHeight() - 2; _width = terminal.getWidth() - 2; @@ -380,14 +530,19 @@ public class TableDisplay implements Closeable { } public void endLine() { + _lineBuilder.close(); int pos = _line - _posY; if (pos >= 0 && pos < _height) { int end = _posX + _width; - _builder.append(_lineBuilder.substring(_posX, Math.min(_lineBuilder.length(), end))); - _builder.append('\n'); + int s = _posX; + int e = Math.min(_lineBuilder.visibleLength(), end); + if (e > s) { + _screenBuilder.append(_lineBuilder.substring(s, e)); + } + _screenBuilder.append('\n'); } _line++; - _lineBuilder.setLength(0); + _lineBuilder.reset(); } public void startLine() { @@ -399,15 +554,13 @@ public class TableDisplay implements Closeable { } public void append(String s) { - for (int i = 0; i < s.length(); i++) { - append(s.charAt(i)); - } + _lineBuilder.append(s); } public void write() throws IOException { _reader.clearScreen(); Writer writer = _reader.getOutput(); - writer.write(_builder.toString()); + writer.write(_screenBuilder.toString()); writer.flush(); }
