The document fetch is now working.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/46d9a00c Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/46d9a00c Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/46d9a00c Branch: refs/heads/0.2-dev Commit: 46d9a00c36402ef8b86742753e354c00bd9cfebd Parents: 68cf6ed Author: Aaron McCurry <[email protected]> Authored: Mon Nov 26 21:46:21 2012 -0500 Committer: Aaron McCurry <[email protected]> Committed: Mon Nov 26 21:46:21 2012 -0500 ---------------------------------------------------------------------- .../org/apache/blur/thrift/BlurShardServer.java | 36 +++++++++++---- .../main/java/org/apache/blur/utils/BlurUtil.java | 20 +++++--- .../apache/blur/utils/ThriftLuceneConversion.java | 24 +++++++++- .../org/apache/blur/thrift/ITBlurClusterTest.java | 11 +++- .../blur/testsuite/AddDocumentsLuceneApiTable.java | 2 +- .../org/apache/blur/testsuite/CreateTable.java | 19 +++----- 6 files changed, 78 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/46d9a00c/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java index d362653..2fe9aca 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java @@ -19,6 +19,8 @@ package org.apache.blur.thrift; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_QUERYCACHE_ELEMENTS; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_CACHE_MAX_TIMETOLIVE; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_DATA_FETCH_THREAD_COUNT; +import static org.apache.blur.utils.ThriftLuceneConversion.setShardIndexTopDocs; +import static org.apache.blur.utils.ThriftLuceneConversion.setShardIndexTopFieldDocs; import static org.apache.blur.utils.ThriftLuceneConversion.toThrift; import java.io.IOException; @@ -294,26 +296,22 @@ public class BlurShardServer extends TableAdmin implements Iface { private TopFieldDocs addShardIndex(TopFieldDocs topFieldDocs) { topFieldDocs.setShardIndex(shardIndex); - List<org.apache.blur.thrift.generated.ScoreDoc> scoreDocs = topFieldDocs.getScoreDocs(); - for (org.apache.blur.thrift.generated.ScoreDoc sd : scoreDocs) { - long docLocation = sd.getDocLocation(); - // docLocation.setShardIndex(shardIndex); - } return topFieldDocs; } private TopFieldDocs doSearch() throws IOException { if (after == null) { if (sort == null) { - return toThrift(searcher.search(query, filter, count)); + return toThrift(setShardIndexTopDocs(shardIndex, searcher.search(query, filter, count))); } else { - return toThrift(searcher.search(query, filter, count, sort, doDocScores, doMaxScore)); + return toThrift(setShardIndexTopFieldDocs(shardIndex, searcher.search(query, filter, count, sort, doDocScores, doMaxScore))); } } else { if (sort == null) { - return toThrift(searcher.searchAfter(after, query, filter, count)); + return toThrift(setShardIndexTopDocs(shardIndex, searcher.searchAfter(after, query, filter, count))); } else { - return toThrift((org.apache.lucene.search.TopFieldDocs) searcher.searchAfter(after, query, filter, count, sort, doDocScores, doMaxScore)); + return toThrift(setShardIndexTopFieldDocs(shardIndex, + (org.apache.lucene.search.TopFieldDocs) searcher.searchAfter(after, query, filter, count, sort, doDocScores, doMaxScore))); } } } @@ -382,7 +380,11 @@ public class BlurShardServer extends TableAdmin implements Iface { int shardIndex = BlurUtil.getShardIndex(docLocation); int docId = BlurUtil.getDocumentId(docLocation); IndexSearcher searcher = searchers.get(shardIndex); - result.add(toThrift(searcher.document(docId, fieldsToLoad))); + if (searcher == null) { + result.addAll(forwardDoc(session, shardIndex, docLocation, fieldsToLoad)); + } else { + result.add(toThrift(searcher.document(docId, fieldsToLoad))); + } } return result; } catch (Throwable t) { @@ -391,6 +393,20 @@ public class BlurShardServer extends TableAdmin implements Iface { } } + private List<Document> forwardDoc(final Session session, int shardIndex, final Long docLocation, final Set<String> fieldsToLoad) throws BlurException, TException, IOException { + // TODO Make more efficient by making a single call to a server for many + // docLocations + String table = session.getTableName(); + System.out.println(table + " " + shardIndex); + Connection connection = getConnection(table, shardIndex); + return BlurClientManager.execute(connection, new BlurCommand<List<Document>>() { + @Override + public List<Document> call(Client client) throws BlurException, TException { + return client.doc(session, Arrays.asList(docLocation), fieldsToLoad); + } + }); + } + @Override public void closeReadSession(Session session) throws BlurException, TException { SessionInfo sessionInfo = getSessionInfo(session); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/46d9a00c/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java index 7ba2eed..b9cb4cc 100644 --- a/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java +++ b/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java @@ -42,6 +42,7 @@ import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants; import org.apache.blur.metrics.BlurMetrics; import org.apache.blur.metrics.BlurMetrics.MethodCall; import org.apache.blur.thrift.generated.Blur.Iface; +import org.apache.blur.thrift.generated.ScoreDoc; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -119,7 +120,6 @@ public class BlurUtil { } } - public static void createIfMissing(ZooKeeper zookeeper, String path) throws KeeperException, InterruptedException { if (zookeeper.exists(path, false) == null) { try { @@ -497,18 +497,24 @@ public class BlurUtil { } public static int getShardIndex(long docLocation) { - // TODO Auto-generated method stub - return 0; + return (int) (docLocation >> 32); } public static int getDocumentId(long docLocation) { - // TODO Auto-generated method stub - return 0; + return (int) (docLocation & 0xFFFFFFFF); } public static long getDocLocation(int shardIndex, int doc) { - // TODO Auto-generated method stub - return 0; + long docLocation = (long) shardIndex << 32L; + return docLocation | ((long) doc & 0xFFFFFFFFL); + } + + public static List<Long> getDocLocations(List<ScoreDoc> scoreDocs) { + List<Long> docLocations = new ArrayList<Long>(scoreDocs.size()); + for (ScoreDoc scoreDoc : scoreDocs) { + docLocations.add(scoreDoc.getDocLocation()); + } + return docLocations; } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/46d9a00c/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java b/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java index 543c82c..affa0a6 100644 --- a/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java +++ b/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java @@ -44,6 +44,28 @@ public class ThriftLuceneConversion { return toThrift(new org.apache.lucene.search.TopFieldDocs(topDocs.totalHits, topDocs.scoreDocs, null, Float.NaN)); } + public static TopDocs setShardIndexTopDocs(int shardIndex, TopDocs topDocs) { + org.apache.lucene.search.ScoreDoc[] scoreDocs = topDocs.scoreDocs; + for (int i = 0; i < scoreDocs.length; i++) { + org.apache.lucene.search.ScoreDoc scoreDoc = scoreDocs[i]; + if (scoreDoc != null) { + scoreDoc.shardIndex = shardIndex; + } + } + return topDocs; + } + + public static org.apache.lucene.search.TopFieldDocs setShardIndexTopFieldDocs(int shardIndex, org.apache.lucene.search.TopFieldDocs topDocs) { + org.apache.lucene.search.ScoreDoc[] scoreDocs = topDocs.scoreDocs; + for (int i = 0; i < scoreDocs.length; i++) { + org.apache.lucene.search.ScoreDoc scoreDoc = scoreDocs[i]; + if (scoreDoc != null) { + scoreDoc.shardIndex = shardIndex; + } + } + return topDocs; + } + public static org.apache.lucene.search.ScoreDoc toLucene(org.apache.blur.thrift.generated.ScoreDoc scoreDoc) { if (scoreDoc == null) { return null; @@ -289,7 +311,7 @@ public class ThriftLuceneConversion { public static Query toLuceneQuery(ByteBuffer byteBuffer) throws IOException { DataInputBuffer in = new DataInputBuffer(); - in.reset(byteBuffer.array(),byteBuffer.arrayOffset() + byteBuffer.position() , byteBuffer.remaining()); + in.reset(byteBuffer.array(), byteBuffer.arrayOffset() + byteBuffer.position(), byteBuffer.remaining()); QueryWritable qw = new QueryWritable(); qw.readFields(in); return qw.getQuery(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/46d9a00c/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java b/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java index 28d1144..3a8971e 100644 --- a/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java +++ b/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java @@ -41,6 +41,7 @@ import org.apache.blur.thrift.generated.TYPE; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.thrift.generated.TopFieldDocs; import org.apache.blur.thrift.generated.UpdatePackage; +import org.apache.blur.utils.BlurUtil; import org.apache.blur.utils.ThriftLuceneConversion; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -143,6 +144,10 @@ public class ITBlurClusterTest { long totalHits = 0; for (TopFieldDocs fieldDocs : results) { totalHits += fieldDocs.getTotalHits(); + List<Document> docs = client.doc(session, BlurUtil.getDocLocations(fieldDocs.getScoreDocs()), null); + for (Document document : docs) { + System.out.println(document); + } } assertEquals(SHARD_COUNT * 100, totalHits); } @@ -190,7 +195,7 @@ public class ITBlurClusterTest { } assertEquals(SHARD_COUNT * 100, totalHits); } - + @Test public void testAddDocumentsPerformance() throws BlurException, TException, InterruptedException, IOException { Iface client = getClient(); @@ -227,9 +232,9 @@ public class ITBlurClusterTest { long t2 = System.nanoTime(); client.blockUntilGenerationIsVisible(generations, true); - + long t3 = System.nanoTime(); - + System.out.println("Add time [" + (t2 - t1) / 1000000000.0 + "]"); System.out.println("Block time [" + (t3 - t2) / 1000000000.0 + "]"); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/46d9a00c/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/AddDocumentsLuceneApiTable.java ---------------------------------------------------------------------- diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/AddDocumentsLuceneApiTable.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/AddDocumentsLuceneApiTable.java index 2983b01..2f6be97 100644 --- a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/AddDocumentsLuceneApiTable.java +++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/AddDocumentsLuceneApiTable.java @@ -26,7 +26,7 @@ import java.util.UUID; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.Document; import org.apache.blur.thrift.generated.Field; -import org.apache.blur.thrift.generated.ShardServer.Client; +import org.apache.blur.thrift.generated.Blur.Client; import org.apache.blur.thrift.generated.MutateOptions; import org.apache.blur.thrift.generated.TYPE; import org.apache.lucene.queryparser.classic.ParseException; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/46d9a00c/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateTable.java ---------------------------------------------------------------------- diff --git a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateTable.java b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateTable.java index 2b4e697..b7d4bda 100644 --- a/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateTable.java +++ b/src/blur-testsuite/src/main/java/org/apache/blur/testsuite/CreateTable.java @@ -19,10 +19,9 @@ package org.apache.blur.testsuite; import java.io.IOException; import org.apache.blur.thrift.BlurClient; -import org.apache.blur.thrift.generated.AnalyzerDefinition; +import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.TableDescriptor; -import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.thrift.TException; @@ -30,19 +29,15 @@ public class CreateTable { public static void main(String[] args) throws BlurException, TException, IOException { String connectionStr = args[0]; - final String cluster = args[1]; - final String tableName = args[2]; - int shardCount = Integer.parseInt(args[3]); - String uri = args[4]; + final String tableName = args[1]; + int shardCount = Integer.parseInt(args[2]); + String uri = args[3]; final TableDescriptor tableDescriptor = new TableDescriptor(); - tableDescriptor.analyzerDefinition = new AnalyzerDefinition(); - tableDescriptor.cluster = cluster; - tableDescriptor.name = tableName; - tableDescriptor.readOnly = false; + tableDescriptor.setName ( tableName); - tableDescriptor.shardCount = shardCount; - tableDescriptor.tableUri = uri; + tableDescriptor.setShardCount(shardCount); + tableDescriptor.setStoragePath( uri); Iface client = BlurClient.getClient(connectionStr); client.createTable(tableDescriptor);
