Fixed the code to the API changes as well as fixing the behaviors and adding unit tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/7315e127 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/7315e127 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/7315e127 Branch: refs/heads/master Commit: 7315e1277468c5d61b8a0f9d831dca482df5ada0 Parents: da6c233 Author: Aaron McCurry <[email protected]> Authored: Wed Jan 15 15:24:42 2014 -0500 Committer: Aaron McCurry <[email protected]> Committed: Wed Jan 15 17:02:52 2014 -0500 ---------------------------------------------------------------------- .../org/apache/blur/manager/IndexManager.java | 20 ++- .../blur/manager/writer/MutatableAction.java | 21 ++- .../blur/thrift/BlurControllerServer.java | 8 +- .../java/org/apache/blur/utils/BlurUtil.java | 15 +- .../org/apache/blur/utils/RowDocumentUtil.java | 1 - .../test/java/org/apache/blur/MiniCluster.java | 1 - .../apache/blur/manager/IndexManagerTest.java | 150 ++++++++++++------- .../org/apache/blur/thrift/BlurClusterTest.java | 1 - .../org/apache/blur/utils/BlurUtilsTest.java | 22 ++- .../org/apache/blur/mapreduce/BlurReducer.java | 4 +- .../org/apache/blur/shell/GetRowCommand.java | 15 +- .../apache/blur/shell/LoadTestDataCommand.java | 7 +- .../org/apache/blur/shell/QueryCommand.java | 2 +- .../org/apache/blur/shell/SelectorCommand.java | 7 +- .../thrift/util/CreateInsertQueryRepeating.java | 2 - .../org/apache/blur/thrift/util/LoadData.java | 6 +- .../apache/blur/thrift/util/LoadDataAsync.java | 10 +- .../thrift/util/LoadDataAsyncContinuously.java | 1 - .../blur/thrift/util/LoadDataContinuously.java | 1 - .../util/RapidlyCreateAndDeleteTables.java | 1 - 20 files changed, 176 insertions(+), 119 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java index 2d49a61..66689df 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java +++ b/blur-core/src/main/java/org/apache/blur/manager/IndexManager.java @@ -41,6 +41,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLongArray; import org.apache.blur.analysis.FieldManager; @@ -683,13 +684,12 @@ public class IndexManager { if (rowId == null) { rowId = getRowId(reader, docId); } - Term term = new Term(ROW_ID, rowId); - int recordCount = BlurUtil.countDocuments(reader, term); fetchResult.rowResult = new FetchRowResult(); - fetchResult.rowResult.row = new Row(rowId, null, recordCount); + fetchResult.rowResult.row = new Row(rowId, null); } else { List<Document> docs; AtomicBoolean moreDocsToFetch = new AtomicBoolean(false); + AtomicInteger totalRecords = new AtomicInteger(); if (highlightQuery != null && fieldManager != null) { String rowId = selector.getRowId(); if (rowId == null) { @@ -706,12 +706,20 @@ public class IndexManager { } else { Tracer docTrace = Trace.trace("fetchRow - Document read"); docs = BlurUtil.fetchDocuments(reader, fieldVisitor, selector, maxHeap, table + "/" + shard, - tableContext.getDefaultPrimeDocTerm(), filter, moreDocsToFetch); + tableContext.getDefaultPrimeDocTerm(), filter, moreDocsToFetch, totalRecords); docTrace.done(); } Tracer rowTrace = Trace.trace("fetchRow - Row create"); - fetchResult.rowResult = new FetchRowResult(getRow(docs), selector.getStartRecord(), - selector.getMaxRecordsToFetch(), moreDocsToFetch.get()); + Row row = getRow(docs); + if (row == null) { + String rowId = selector.getRowId(); + if (rowId == null) { + rowId = getRowId(reader, docId); + } + row = new Row(rowId, null); + } + fetchResult.rowResult = new FetchRowResult(row, selector.getStartRecord(), selector.getMaxRecordsToFetch(), + moreDocsToFetch.get(), totalRecords.get()); rowTrace.done(); } return; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-core/src/main/java/org/apache/blur/manager/writer/MutatableAction.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/MutatableAction.java b/blur-core/src/main/java/org/apache/blur/manager/writer/MutatableAction.java index 4e91c1c..3e09823 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/MutatableAction.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/MutatableAction.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.blur.analysis.FieldManager; import org.apache.blur.manager.IndexManager; @@ -113,8 +114,9 @@ public class MutatableAction { selector.setLocationId(_shard + "/" + topDocs.scoreDocs[0].doc); ResetableDocumentStoredFieldVisitor fieldVisitor = IndexManager.getFieldSelector(selector); AtomicBoolean moreDocsToFetch = new AtomicBoolean(false); + AtomicInteger totalRecords = new AtomicInteger(); List<Document> docs = new ArrayList<Document>(BlurUtil.fetchDocuments(reader, fieldVisitor, selector, - _maxHeap, _table + "/" + _shard, _primeDocTerm, null, moreDocsToFetch)); + _maxHeap, _table + "/" + _shard, _primeDocTerm, null, moreDocsToFetch, totalRecords)); if (moreDocsToFetch.get()) { throw new IOException("Row too large to update."); } @@ -131,7 +133,7 @@ public class MutatableAction { if (docs.size() == 0) { writer.deleteDocuments(rowIdTerm); } else { - Row row = new Row(rowId, toRecords(docs), docs.size()); + Row row = new Row(rowId, toRecords(docs)); List<List<Field>> docsToUpdate = RowDocumentUtil.getDocs(row, _fieldManager); writer.updateDocuments(rowIdTerm, docsToUpdate); } @@ -166,8 +168,9 @@ public class MutatableAction { selector.setLocationId(_shard + "/" + topDocs.scoreDocs[0].doc); ResetableDocumentStoredFieldVisitor fieldVisitor = IndexManager.getFieldSelector(selector); AtomicBoolean moreDocsToFetch = new AtomicBoolean(false); + AtomicInteger totalRecords = new AtomicInteger(); List<Document> docs = new ArrayList<Document>(BlurUtil.fetchDocuments(reader, fieldVisitor, selector, - _maxHeap, _table + "/" + _shard, _primeDocTerm, null, moreDocsToFetch)); + _maxHeap, _table + "/" + _shard, _primeDocTerm, null, moreDocsToFetch, totalRecords)); if (moreDocsToFetch.get()) { throw new IOException("Row too large to update."); } @@ -181,7 +184,7 @@ public class MutatableAction { } } docs.add(toDocument(doc)); - Row row = new Row(rowId, toRecords(docs), docs.size()); + Row row = new Row(rowId, toRecords(docs)); List<List<Field>> docsToUpdate = RowDocumentUtil.getDocs(row, _fieldManager); writer.updateDocuments(rowIdTerm, docsToUpdate); } else { @@ -224,8 +227,9 @@ public class MutatableAction { selector.setLocationId(_shard + "/" + topDocs.scoreDocs[0].doc); ResetableDocumentStoredFieldVisitor fieldVisitor = IndexManager.getFieldSelector(selector); AtomicBoolean moreDocsToFetch = new AtomicBoolean(false); + AtomicInteger totalRecords = new AtomicInteger(); List<Document> docs = new ArrayList<Document>(BlurUtil.fetchDocuments(reader, fieldVisitor, selector, - _maxHeap, _table + "/" + _shard, _primeDocTerm, null, moreDocsToFetch)); + _maxHeap, _table + "/" + _shard, _primeDocTerm, null, moreDocsToFetch, totalRecords)); if (moreDocsToFetch.get()) { throw new IOException("Row too large to update."); } @@ -260,7 +264,7 @@ public class MutatableAction { List<Field> doc = RowDocumentUtil.getDoc(_fieldManager, rowId, existingRecord); docs.add(toDocument(doc)); - Row row = new Row(rowId, toRecords(docs), docs.size()); + Row row = new Row(rowId, toRecords(docs)); List<List<Field>> docsToUpdate = RowDocumentUtil.getDocs(row, _fieldManager); writer.updateDocuments(rowIdTerm, docsToUpdate); @@ -294,8 +298,9 @@ public class MutatableAction { selector.setLocationId(_shard + "/" + topDocs.scoreDocs[0].doc); ResetableDocumentStoredFieldVisitor fieldVisitor = IndexManager.getFieldSelector(selector); AtomicBoolean moreDocsToFetch = new AtomicBoolean(false); + AtomicInteger totalRecords = new AtomicInteger(); List<Document> docs = new ArrayList<Document>(BlurUtil.fetchDocuments(reader, fieldVisitor, selector, - _maxHeap, _table + "/" + _shard, _primeDocTerm, null, moreDocsToFetch)); + _maxHeap, _table + "/" + _shard, _primeDocTerm, null, moreDocsToFetch, totalRecords)); if (moreDocsToFetch.get()) { throw new IOException("Row too large to update."); } @@ -356,7 +361,7 @@ public class MutatableAction { List<Field> doc = RowDocumentUtil.getDoc(_fieldManager, rowId, newRecord); docs.add(toDocument(doc)); - Row row = new Row(rowId, toRecords(docs), docs.size()); + Row row = new Row(rowId, toRecords(docs)); List<List<Field>> docsToUpdate = RowDocumentUtil.getDocs(row, _fieldManager); writer.updateDocuments(rowIdTerm, docsToUpdate); } else { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java index f02490c..696b947 100644 --- a/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java +++ b/blur-core/src/main/java/org/apache/blur/thrift/BlurControllerServer.java @@ -22,6 +22,7 @@ import java.net.Socket; import java.net.SocketAddress; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.IdentityHashMap; @@ -147,8 +148,9 @@ public class BlurControllerServer extends TableAdmin implements Iface { private static final String CONTROLLER_THREAD_POOL = "controller-thread-pool"; private static final Log LOG = LogFactory.getLog(BlurControllerServer.class); - private static final Map<String, Set<String>> EMPTY_MAP = new HashMap<String, Set<String>>(); - private static final List<String> EMPTY_LIST = new ArrayList<String>(); + private static final Map<String, Set<String>> EMPTY_MAP = Collections + .unmodifiableMap(new HashMap<String, Set<String>>()); + private static final Set<String> EMPTY_SET = Collections.unmodifiableSet(new HashSet<String>()); private ExecutorService _executor; private AtomicReference<Map<String, Map<String, String>>> _shardServerLayout = new AtomicReference<Map<String, Map<String, String>>>( @@ -375,7 +377,7 @@ public class BlurControllerServer extends TableAdmin implements Iface { Selector selector = blurQuery.getSelector(); if (selector == null) { selector = new Selector(); - selector.setColumnFamiliesToFetch(EMPTY_LIST); + selector.setColumnFamiliesToFetch(EMPTY_SET); selector.setColumnsToFetch(EMPTY_MAP); if (!blurQuery.query.rowQuery) { selector.setRecordOnly(true); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java index 3f14ddc..01271bf 100644 --- a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java +++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java @@ -56,6 +56,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLongArray; import java.util.regex.Pattern; @@ -798,12 +799,13 @@ public class BlurUtil { * @param selector * @param primeDocTerm * @param filter + * @param totalRecords * * @throws IOException */ @SuppressWarnings("unchecked") public static List<Document> fetchDocuments(IndexReader reader, ResetableDocumentStoredFieldVisitor fieldSelector, - Selector selector, int maxHeap, String context, Term primeDocTerm, Filter filter, AtomicBoolean moreToFetch) + Selector selector, int maxHeap, String context, Term primeDocTerm, Filter filter, AtomicBoolean moreToFetch, AtomicInteger totalRecords) throws IOException { if (reader instanceof BaseCompositeReader) { BaseCompositeReader<IndexReader> indexReader = (BaseCompositeReader<IndexReader>) reader; @@ -832,7 +834,7 @@ public class BlurUtil { numberOfDocsInRow = nextPrimeDoc - primeDocId; } OpenBitSet docsInRowSpanToFetch = getDocsToFetch(segmentReader, selector, primeDocId, numberOfDocsInRow, - liveDocs, filter); + liveDocs, filter, totalRecords); int start = selector.getStartRecord(); int maxDocsToFetch = selector.getMaxRecordsToFetch(); int startingPosition = getStartingPosition(docsInRowSpanToFetch, start); @@ -877,7 +879,8 @@ public class BlurUtil { } private static List<Document> orderDocsBasedOnFamilyOrder(List<Document> docs, Selector selector) { - List<String> columnFamiliesToFetch = selector.getColumnFamiliesToFetch(); + List<String> columnFamiliesToFetch = selector.getColumnFamiliesToFetch() == null ? null : new ArrayList<String>( + selector.getColumnFamiliesToFetch()); if (columnFamiliesToFetch == null || columnFamiliesToFetch.isEmpty()) { return docs; } @@ -935,7 +938,7 @@ public class BlurUtil { } private static OpenBitSet getDocsToFetch(SegmentReader segmentReader, Selector selector, int primeDocRowId, - int numberOfDocsInRow, Bits liveDocs, Filter filter) throws IOException { + int numberOfDocsInRow, Bits liveDocs, Filter filter, AtomicInteger totalRecords) throws IOException { Set<String> alreadyProcessed = new HashSet<String>(); OpenBitSet bits = new OpenBitSet(numberOfDocsInRow); OpenBitSet mask = null; @@ -943,7 +946,8 @@ public class BlurUtil { DocIdSet docIdSet = filter.getDocIdSet(segmentReader.getContext(), liveDocs); mask = getMask(docIdSet, primeDocRowId, numberOfDocsInRow); } - List<String> columnFamiliesToFetch = selector.getColumnFamiliesToFetch(); + List<String> columnFamiliesToFetch = selector.getColumnFamiliesToFetch() == null ? null : new ArrayList<String>( + selector.getColumnFamiliesToFetch()); boolean fetchAll = true; if (columnFamiliesToFetch != null) { fetchAll = false; @@ -961,6 +965,7 @@ public class BlurUtil { if (mask != null) { bits.intersect(mask); } + totalRecords.set((int) bits.cardinality()); return bits; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java b/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java index 97d89b0..085147c 100644 --- a/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java +++ b/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java @@ -71,7 +71,6 @@ public class RowDocumentUtil { if (row.id == null) { row.setId(rowId); } - row.recordCount++; } if (empty) { return null; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-core/src/test/java/org/apache/blur/MiniCluster.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/MiniCluster.java b/blur-core/src/test/java/org/apache/blur/MiniCluster.java index 58535cb..3a2f826 100644 --- a/blur-core/src/test/java/org/apache/blur/MiniCluster.java +++ b/blur-core/src/test/java/org/apache/blur/MiniCluster.java @@ -203,7 +203,6 @@ public class MiniCluster { record.addToColumns(new Column("test", Integer.toString(i))); row.addToRecords(record); RowMutation rowMutation = BlurUtil.toRowMutation(table, row); - rowMutation.setWal(false); client.mutate(rowMutation); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java index e805930..68ad8ea 100644 --- a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java +++ b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java @@ -58,6 +58,7 @@ import org.apache.blur.thrift.generated.Column; import org.apache.blur.thrift.generated.Facet; import org.apache.blur.thrift.generated.FetchRecordResult; import org.apache.blur.thrift.generated.FetchResult; +import org.apache.blur.thrift.generated.FetchRowResult; import org.apache.blur.thrift.generated.HighlightOptions; import org.apache.blur.thrift.generated.Query; import org.apache.blur.thrift.generated.Record; @@ -267,7 +268,6 @@ public class IndexManagerTest { RowMutation mutation7 = newRowMutation(TABLE, "row-7", newRecordMutation(FAMILY, "record-7A", newColumn("testcol12", "value101"), newColumn("testcol13", "value102")), newRecordMutation(FAMILY2, "record-7B", newColumn("testcol18", "value501"))); - mutation7.waitToBeVisible = true; indexManager.mutate(mutation1); indexManager.mutate(mutation2); indexManager.mutate(mutation3); @@ -295,28 +295,35 @@ public class IndexManagerTest { } }); + List<Thread> threads = new ArrayList<Thread>(); for (int i = 0; i < 25; i++) { Thread thread = new Thread(new Runnable() { @Override public void run() { - // Trace.setupTrace(rowId); - Selector selector = new Selector().setRowId(rowId); - FetchResult fetchResult = new FetchResult(); - long s = System.nanoTime(); - try { - indexManager.fetchRow(TABLE, selector, fetchResult); - } catch (BlurException e1) { - e1.printStackTrace(); + for (int t = 0; t < 50; t++) { + // Trace.setupTrace(rowId); + Selector selector = new Selector().setRowId(rowId); + FetchResult fetchResult = new FetchResult(); + long s = System.nanoTime(); + try { + indexManager.fetchRow(TABLE, selector, fetchResult); + } catch (BlurException e1) { + e1.printStackTrace(); + } + long e = System.nanoTime(); + assertNotNull(fetchResult.rowResult.row); + // Trace.tearDownTrace(); + System.out.println((e - s) / 1000000.0); } - long e = System.nanoTime(); - assertNotNull(fetchResult.rowResult.row); - // Trace.tearDownTrace(); - System.out.println((e - s) / 1000000.0); } }); - + threads.add(thread); thread.start(); - thread.join(); + // thread.join(); + } + + for (Thread t : threads) { + t.join(); } Trace.setStorage(oldReporter); @@ -327,8 +334,6 @@ public class IndexManagerTest { RowMutation rowMutation = new RowMutation(); rowMutation.setTable(TABLE); rowMutation.setRowId(rowId); - rowMutation.setWal(true); - rowMutation.setWaitToBeVisible(true); rowMutation.setRecordMutations(getRecordMutations(10000)); rowMutation.setRowMutationType(RowMutationType.REPLACE_ROW); return rowMutation; @@ -369,8 +374,9 @@ public class IndexManagerTest { assertNotNull(fetchResult.rowResult.row); Row row = newRow("row-6", newRecord(FAMILY, "record-6B", newColumn("testcol12", "<<<value101>>>"))); - row.recordCount = 3; - assertEquals(row, fetchResult.rowResult.row); + FetchRowResult rowResult = fetchResult.getRowResult(); + assertEquals(row, rowResult.getRow()); + assertEquals(3, rowResult.getTotalRecords()); } @Test @@ -386,8 +392,9 @@ public class IndexManagerTest { assertNotNull(fetchResult.rowResult.row); Row row = newRow("row-6", newRecord(FAMILY, "record-6B", newColumn("testcol12", "<<<value101>>>"))); - row.recordCount = 3; - assertEquals(row, fetchResult.rowResult.row); + FetchRowResult rowResult = fetchResult.getRowResult(); + assertEquals(row, rowResult.getRow()); + assertEquals(3, rowResult.getTotalRecords()); } @Test @@ -403,8 +410,10 @@ public class IndexManagerTest { assertNotNull(fetchResult.rowResult.row); Row row = newRow("row-6", newRecord(FAMILY, "record-6B", newColumn("testcol12", "<<<value101>>>"))); - row.recordCount = 3; - assertEquals(row, fetchResult.rowResult.row); + + FetchRowResult rowResult = fetchResult.getRowResult(); + assertEquals(row, rowResult.getRow()); + assertEquals(3, rowResult.getTotalRecords()); } @Test @@ -550,8 +559,10 @@ public class IndexManagerTest { "row-1", newRecord(FAMILY, "record-1", newColumn("testcol1", "value1"), newColumn("testcol2", "value2"), newColumn("testcol3", "value3"))); - row.recordCount = 1; assertEquals(row, fetchResult.rowResult.row); + FetchRowResult rowResult = fetchResult.getRowResult(); + assertEquals(row, rowResult.getRow()); + assertEquals(1, rowResult.getTotalRecords()); } @Test @@ -592,8 +603,21 @@ public class IndexManagerTest { "row-1", newRecord(FAMILY, "record-1", newColumn("testcol1", "value1"), newColumn("testcol2", "value2"), newColumn("testcol3", "value3"))); - row.recordCount = 1; - assertEquals(row, fetchResult.rowResult.row); + FetchRowResult rowResult = fetchResult.getRowResult(); + assertEquals(row, rowResult.getRow()); + assertEquals(1, rowResult.getTotalRecords()); + } + + @Test + public void testFetchRowByRowIdWithInvalidFamily() throws Exception { + Selector selector = new Selector().setRowId("row-1"); + selector.addToColumnFamiliesToFetch(UUID.randomUUID().toString()); + FetchResult fetchResult = new FetchResult(); + indexManager.fetchRow(TABLE, selector, fetchResult); + assertNotNull(fetchResult.rowResult.row); + FetchRowResult rowResult = fetchResult.getRowResult(); + assertEquals(new Row("row-1", null), rowResult.getRow()); + assertEquals(0, rowResult.getTotalRecords()); } @Test @@ -604,8 +628,9 @@ public class IndexManagerTest { indexManager.fetchRow(TABLE, selector, fetchResult); assertNotNull(fetchResult.rowResult.row); Row row = newRow("row-6", newRecord(FAMILY2, "record-6C", newColumn("testcol18", "value501"))); - row.recordCount = 1; - assertEquals(row, fetchResult.rowResult.row); + FetchRowResult rowResult = fetchResult.getRowResult(); + assertEquals(row, rowResult.getRow()); + assertEquals(1, rowResult.getTotalRecords()); } @Test @@ -618,8 +643,9 @@ public class IndexManagerTest { assertNotNull(fetchResult.rowResult.row); Row row = newRow("row-6", newRecord(FAMILY, "record-6A", newColumn("testcol12", "value110")), newRecord(FAMILY, "record-6B", newColumn("testcol12", "value101"))); - row.recordCount = 2; - assertEquals(row, fetchResult.rowResult.row); + FetchRowResult rowResult = fetchResult.getRowResult(); + assertEquals(row, rowResult.getRow()); + assertEquals(2, rowResult.getTotalRecords()); } @Test @@ -634,8 +660,9 @@ public class IndexManagerTest { Row row = newRow("row-6", newRecord(FAMILY2, "record-6C", newColumn("testcol18", "value501")), newRecord(FAMILY, "record-6A", newColumn("testcol12", "value110")), newRecord(FAMILY, "record-6B", newColumn("testcol12", "value101"))); - row.recordCount = 3; - assertEquals(row, fetchResult.rowResult.row); + FetchRowResult rowResult = fetchResult.getRowResult(); + assertEquals(row, rowResult.getRow()); + assertEquals(3, rowResult.getTotalRecords()); } @Test @@ -652,8 +679,10 @@ public class IndexManagerTest { assertNotNull(fetchResult.rowResult.row); Row row = newRow("row-6", newRecord(FAMILY, "record-6A", newColumn("testcol12", "value110"), newColumn("testcol13", "value102"))); - row.recordCount = 1; - assertEquals(row, fetchResult.rowResult.row); + + FetchRowResult rowResult = fetchResult.getRowResult(); + assertEquals(row, rowResult.getRow()); + assertEquals(1, rowResult.getTotalRecords()); } @Test @@ -669,7 +698,7 @@ public class IndexManagerTest { indexManager.fetchRow(TABLE, selector, fetchResult); assertTrue(fetchResult.exists); assertFalse(fetchResult.deleted); - assertNull(fetchResult.rowResult.row); + assertNull(fetchResult.rowResult.row.records); } @Test @@ -685,8 +714,10 @@ public class IndexManagerTest { "row-1", newRecord(FAMILY, "record-1", newColumn("testcol1", "value1"), newColumn("testcol2", "value2"), newColumn("testcol3", "value3"))); - row1.recordCount = 1; - assertEquals(row1, fetchResult1.rowResult.row); + + FetchRowResult rowResult1 = fetchResult1.getRowResult(); + assertEquals(row1, rowResult1.getRow()); + assertEquals(1, rowResult1.getTotalRecords()); FetchResult fetchResult2 = fetchRowBatch.get(1); assertNotNull(fetchResult2.rowResult.row); @@ -695,8 +726,10 @@ public class IndexManagerTest { newRecord(FAMILY, "record-2", newColumn("testcol1", "value4"), newColumn("testcol2", "value5"), newColumn("testcol3", "value6")), newRecord(FAMILY, "record-2B", newColumn("testcol2", "value234123"), newColumn("testcol3", "value234123"))); - row2.recordCount = 2; - assertEquals(row2, fetchResult2.rowResult.row); + + FetchRowResult rowResult2 = fetchResult2.getRowResult(); + assertEquals(row2, rowResult2.getRow()); + assertEquals(2, rowResult2.getTotalRecords()); } @Test @@ -720,8 +753,9 @@ public class IndexManagerTest { Row row1 = newRow("row-6", newRecord(FAMILY, "record-6A", newColumn("testcol12", "value110"), newColumn("testcol13", "value102"))); - row1.recordCount = 1; - assertEquals(row1, fetchResult.rowResult.row); + FetchRowResult rowResult = fetchResult.getRowResult(); + assertEquals(row1, rowResult.getRow()); + assertEquals(3, rowResult.getTotalRecords()); selector = new Selector().setRowId("row-6").setStartRecord(1).setMaxRecordsToFetch(1); fetchResult = new FetchResult(); @@ -733,8 +767,9 @@ public class IndexManagerTest { Row row2 = newRow("row-6", newRecord(FAMILY, "record-6B", newColumn("testcol12", "value101"), newColumn("testcol13", "value104"))); - row2.recordCount = 1; - assertEquals(row2, fetchResult.rowResult.row); + FetchRowResult rowResult2 = fetchResult.getRowResult(); + assertEquals(row2, rowResult2.getRow()); + assertEquals(3, rowResult2.getTotalRecords()); selector = new Selector().setRowId("row-6").setStartRecord(2).setMaxRecordsToFetch(1); fetchResult = new FetchResult(); @@ -745,13 +780,16 @@ public class IndexManagerTest { assertEquals(1, fetchResult.rowResult.maxRecordsToFetch); Row row3 = newRow("row-6", newRecord(FAMILY2, "record-6C", newColumn("testcol18", "value501"))); - row3.recordCount = 1; - assertEquals(row3, fetchResult.rowResult.row); + + FetchRowResult rowResult3 = fetchResult.getRowResult(); + assertEquals(row3, rowResult3.getRow()); + assertEquals(3, rowResult3.getTotalRecords()); selector = new Selector().setRowId("row-6").setStartRecord(3).setMaxRecordsToFetch(1); fetchResult = new FetchResult(); indexManager.fetchRow(TABLE, selector, fetchResult); - assertNull(fetchResult.rowResult.row); + assertNull(fetchResult.rowResult.row.records); + assertEquals(3, fetchResult.rowResult.getTotalRecords()); } @Test @@ -1092,7 +1130,6 @@ public class IndexManagerTest { "row-4", newRecordMutation(FAMILY, "record-4", newColumn("testcol1", "value2"), newColumn("testcol2", "value3"), newColumn("testcol3", "value4"))); - mutation.waitToBeVisible = true; indexManager.mutate(mutation); Selector selector = new Selector().setRowId("row-4"); @@ -1103,8 +1140,10 @@ public class IndexManagerTest { "row-4", newRecord(FAMILY, "record-4", newColumn("testcol1", "value2"), newColumn("testcol2", "value3"), newColumn("testcol3", "value4"))); - row.recordCount = 1; - assertEquals(row, fetchResult.rowResult.row); + + FetchRowResult rowResult = fetchResult.getRowResult(); + assertEquals(row, rowResult.getRow()); + assertEquals(1, rowResult.getTotalRecords()); } @Test @@ -1115,7 +1154,6 @@ public class IndexManagerTest { String rec = "record-6"; RecordMutation rm = newRecordMutation(FAMILY, rec, c1, c2, c3); RowMutation mutation = newRowMutation(TABLE, "row-6", rm); - mutation.waitToBeVisible = true; indexManager.mutate(mutation); Selector selector = new Selector().setRowId("row-6"); @@ -1127,14 +1165,15 @@ public class IndexManagerTest { "row-6", newRecord(FAMILY, "record-6", newColumn("testcol1", "value20"), newColumn("testcol2", "value21"), newColumn("testcol3", "value22"))); - row.recordCount = 1; - assertEquals("row should match", row, r); + + FetchRowResult rowResult = fetchResult.getRowResult(); + assertEquals(row, rowResult.getRow()); + assertEquals(1, rowResult.getTotalRecords()); } @Test public void testMutationDeleteRow() throws Exception { RowMutation mutation = newRowMutation(DELETE_ROW, TABLE, "row-2"); - mutation.waitToBeVisible = true; indexManager.mutate(mutation); Selector selector = new Selector().setRowId("row-2"); @@ -1146,7 +1185,6 @@ public class IndexManagerTest { @Test public void testMutationDeleteMissingRow() throws Exception { RowMutation mutation = newRowMutation(DELETE_ROW, TABLE, "row-6"); - mutation.waitToBeVisible = true; indexManager.mutate(mutation); Selector selector = new Selector().setRowId("row-6"); @@ -1161,7 +1199,6 @@ public class IndexManagerTest { RowMutation rowMutation = newRowMutation(UPDATE_ROW, TABLE, "row-3", rm); - rowMutation.waitToBeVisible = true; indexManager.mutate(rowMutation); Selector selector = new Selector().setRowId("row-3"); @@ -1175,7 +1212,6 @@ public class IndexManagerTest { RecordMutation rm = newRecordMutation(DELETE_ENTIRE_RECORD, FAMILY, "record-5A"); RowMutation rowMutation = newRowMutation(UPDATE_ROW, TABLE, "row-5", rm); - rowMutation.waitToBeVisible = true; indexManager.mutate(rowMutation); Selector selector = new Selector().setRowId("row-5"); @@ -1232,7 +1268,6 @@ public class IndexManagerTest { RecordMutation rm2 = newRecordMutation(REPLACE_ENTIRE_RECORD, FAMILY, "record-5C", c4, c5, c6); RowMutation rowMutation = newRowMutation(UPDATE_ROW, TABLE, "row-5", rm1, rm2); - rowMutation.waitToBeVisible = true; indexManager.mutate(rowMutation); Selector selector = new Selector().setRowId("row-5"); @@ -1455,7 +1490,6 @@ public class IndexManagerTest { private Record updateAndFetchRecord(String rowId, String recordId, RecordMutation... recordMutations) throws Exception { RowMutation rowMutation = newRowMutation(UPDATE_ROW, TABLE, rowId, recordMutations); - rowMutation.waitToBeVisible = true; indexManager.mutate(rowMutation); Selector selector = new Selector().setRowId(rowId).setRecordId(recordId); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java index 66d4372..336e8b3 100644 --- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java +++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java @@ -212,7 +212,6 @@ public class BlurClusterTest { BlurThriftHelper.newColumn("test", "value"), BlurThriftHelper.newColumn("facet", Integer.toString(random.nextInt(maxFacetValue)))); RowMutation rowMutation = BlurThriftHelper.newRowMutation("test", rowId, mutation); - rowMutation.setWaitToBeVisible(true); mutations.add(rowMutation); } long s = System.nanoTime(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java b/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java index 27c31e8..4e2b697 100644 --- a/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java +++ b/blur-core/src/test/java/org/apache/blur/utils/BlurUtilsTest.java @@ -24,10 +24,12 @@ import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; -import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.blur.thrift.generated.Selector; import org.apache.hadoop.conf.Configuration; @@ -165,7 +167,7 @@ public class BlurUtilsTest { public void testFetchDocuments() throws CorruptIndexException, LockObtainFailedException, IOException { Selector selector = new Selector(); selector.setLocationId("shard/0"); - List<String> columnFamiliesToFetch = new ArrayList<String>(); + Set<String> columnFamiliesToFetch = new HashSet<String>(); columnFamiliesToFetch.add("f1"); columnFamiliesToFetch.add("f2"); selector.setColumnFamiliesToFetch(columnFamiliesToFetch); @@ -176,10 +178,13 @@ public class BlurUtilsTest { // "test-context", new // Term(BlurConstants.PRIME_DOC,BlurConstants.PRIME_DOC_VALUE)); AtomicBoolean moreDocsToFetch = new AtomicBoolean(false); + AtomicInteger totalRecords = new AtomicInteger(); List<Document> docs = BlurUtil.fetchDocuments(getReader(), resetableDocumentStoredFieldVisitor, selector, 10000000, - "test-context", new Term(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE), null, moreDocsToFetch); + "test-context", new Term(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE), null, moreDocsToFetch, + totalRecords); assertEquals(docs.size(), 1); assertFalse(moreDocsToFetch.get()); + assertEquals(1, totalRecords.get()); } @Test @@ -187,20 +192,22 @@ public class BlurUtilsTest { IOException { Selector selector = new Selector(); selector.setLocationId("shard/0"); - List<String> columnFamiliesToFetch = new ArrayList<String>(); + Set<String> columnFamiliesToFetch = new HashSet<String>(); columnFamiliesToFetch.add("f1"); columnFamiliesToFetch.add("f2"); selector.setColumnFamiliesToFetch(columnFamiliesToFetch); ResetableDocumentStoredFieldVisitor resetableDocumentStoredFieldVisitor = new ResetableDocumentStoredFieldVisitor(); AtomicBoolean moreDocsToFetch = new AtomicBoolean(false); + AtomicInteger totalRecords = new AtomicInteger(); List<Document> docs = BlurUtil.fetchDocuments(getReaderWithDocsHavingFamily(), resetableDocumentStoredFieldVisitor, selector, 10000000, "test-context", new Term(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE), null, - moreDocsToFetch); + moreDocsToFetch, totalRecords); assertEquals(docs.size(), 2); assertEquals(docs.get(0).getField("family").stringValue(), "f1"); assertEquals(docs.get(1).getField("family").stringValue(), "f2"); assertFalse(moreDocsToFetch.get()); + assertEquals(2, totalRecords.get()); } @Test @@ -209,10 +216,13 @@ public class BlurUtilsTest { selector.setLocationId("shard/0"); ResetableDocumentStoredFieldVisitor resetableDocumentStoredFieldVisitor = new ResetableDocumentStoredFieldVisitor(); AtomicBoolean moreDocsToFetch = new AtomicBoolean(false); + AtomicInteger totalRecords = new AtomicInteger(); List<Document> docs = BlurUtil.fetchDocuments(getReader(), resetableDocumentStoredFieldVisitor, selector, 10000000, - "test-context", new Term(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE), null, moreDocsToFetch); + "test-context", new Term(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE), null, moreDocsToFetch, + totalRecords); assertEquals(docs.size(), 2); assertFalse(moreDocsToFetch.get()); + assertEquals(2, totalRecords.get()); } private void rm(File file) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java ---------------------------------------------------------------------- diff --git a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java index b9c5a53..d31c7fb 100644 --- a/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java +++ b/blur-mapred/src/main/java/org/apache/blur/mapreduce/BlurReducer.java @@ -32,6 +32,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.blur.analysis.FieldManager; import org.apache.blur.log.Log; @@ -251,9 +252,10 @@ public class BlurReducer extends Reducer<Text, BlurMutate, Text, BlurMutate> { protected void fetchOldRecords() throws IOException { AtomicBoolean moreDocsToFetch = new AtomicBoolean(false); + AtomicInteger totalRecords = new AtomicInteger(); List<Document> docs = BlurUtil.fetchDocuments(_reader, new ResetableDocumentStoredFieldVisitor(), new Selector() .setRowId(_rowIdTerm.text()), Integer.MAX_VALUE, "reducer-context", new Term(BlurConstants.PRIME_DOC, - BlurConstants.PRIME_DOC_VALUE), null, moreDocsToFetch); + BlurConstants.PRIME_DOC_VALUE), null, moreDocsToFetch, totalRecords); if (moreDocsToFetch.get()) { throw new IOException("Row too large to update."); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-shell/src/main/java/org/apache/blur/shell/GetRowCommand.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/GetRowCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/GetRowCommand.java index 296b364..1ed2d5c 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/GetRowCommand.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/GetRowCommand.java @@ -43,9 +43,9 @@ public class GetRowCommand extends Command implements TableFirstArgCommand { if (args.length != 3) { throw new CommandException("Invalid args: " + help()); } - + PagingPrintWriter out = new PagingPrintWriter(outPw); - + try { doItInternal(client, args, out); } catch (FinishedException e) { @@ -80,15 +80,18 @@ public class GetRowCommand extends Command implements TableFirstArgCommand { maxWidth = terminal.getWidth() - 15; out.setLineLimit(terminal.getHeight() - 2); } - format(out, row, maxWidth); + format(out, rowResult, maxWidth); } - public static void format(PagingPrintWriter out, Row row, int maxWidth) throws FinishedException { + public static void format(PagingPrintWriter out, FetchRowResult rowResult, int maxWidth) throws FinishedException { + Row row = rowResult.getRow(); + if (row == null) { + return; + } String id = row.getId(); - int recordCount = row.getRecordCount(); out.println(" id : " + id); if (Main.debug) { - out.println("recordCount : " + recordCount); + out.println("totalRecords : " + rowResult.getTotalRecords()); } List<Record> records = row.getRecords(); for (Record record : records) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-shell/src/main/java/org/apache/blur/shell/LoadTestDataCommand.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/LoadTestDataCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/LoadTestDataCommand.java index 0b009b2..a4516e2 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/LoadTestDataCommand.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/LoadTestDataCommand.java @@ -30,12 +30,11 @@ public class LoadTestDataCommand extends Command implements TableFirstArgCommand @Override public void doit(PrintWriter out, Blur.Iface client, String[] args) throws CommandException, TException, BlurException { - if (args.length != 9) { + if (args.length != 8) { throw new CommandException("Invalid args: " + help()); } int c = 1; String table = args[c++]; - boolean wal = Boolean.parseBoolean(args[c++]); int numberRows = Integer.parseInt(args[c++]); int numberRecordsPerRow = Integer.parseInt(args[c++]); int numberOfFamilies = Integer.parseInt(args[c++]); @@ -43,7 +42,7 @@ public class LoadTestDataCommand extends Command implements TableFirstArgCommand int numberOfWords = Integer.parseInt(args[c++]); int batch = Integer.parseInt(args[c++]); try { - LoadData.runLoad(client, table, wal, numberRows, numberRecordsPerRow, numberOfFamilies, numberOfColumns, + LoadData.runLoad(client, table, numberRows, numberRecordsPerRow, numberOfFamilies, numberOfColumns, numberOfWords, batch, out); } catch (IOException e) { out.println("Error " + e.getMessage()); @@ -60,7 +59,7 @@ public class LoadTestDataCommand extends Command implements TableFirstArgCommand @Override public String usage() { - return "<tablename> <write ahead log (true/false)> <rows> <recordsPerRow> <families> <columnsPerRecord> <wordsPerColumn> <batchSize>"; + return "<tablename> <rows> <recordsPerRow> <families> <columnsPerRecord> <wordsPerColumn> <batchSize>"; } @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand.java index a8e0f38..b15657a 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand.java @@ -113,7 +113,7 @@ public class QueryCommand extends Command implements TableFirstArgCommand { if (rowResult != null) { Row row = rowResult.getRow(); if (row != null) { - GetRowCommand.format(out, row, maxWidth); + GetRowCommand.format(out, rowResult, maxWidth); } } lineBreak(out, maxWidth); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-shell/src/main/java/org/apache/blur/shell/SelectorCommand.java ---------------------------------------------------------------------- diff --git a/blur-shell/src/main/java/org/apache/blur/shell/SelectorCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/SelectorCommand.java index f59f7ca..0ff52a2 100644 --- a/blur-shell/src/main/java/org/apache/blur/shell/SelectorCommand.java +++ b/blur-shell/src/main/java/org/apache/blur/shell/SelectorCommand.java @@ -21,6 +21,7 @@ package org.apache.blur.shell; import java.io.PrintWriter; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -43,13 +44,13 @@ public class SelectorCommand extends Command { command = args[1]; } if (command == null) { - List<String> columnFamiliesToFetch = Main.selector.getColumnFamiliesToFetch(); + Set<String> columnFamiliesToFetch = Main.selector.getColumnFamiliesToFetch(); Map<String, Set<String>> columnsToFetch = Main.selector.getColumnsToFetch(); List<String> names = new ArrayList<String>(); if (columnFamiliesToFetch != null) { names.addAll(columnFamiliesToFetch); } else { - columnFamiliesToFetch = new ArrayList<String>(); + columnFamiliesToFetch = new HashSet<String>(); } if (columnsToFetch != null) { names.addAll(columnsToFetch.keySet()); @@ -72,7 +73,7 @@ public class SelectorCommand extends Command { String family = args[2]; if (args.length > 3) { if (Main.selector.columnsToFetch == null) { - Main.selector.columnsToFetch = new HashMap<String, Set<String>>(); + Main.selector.columnsToFetch = new HashMap<String, Set<String>>(); } Set<String> columns = Main.selector.columnsToFetch.get(family); if (columns == null) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-thrift/src/main/java/org/apache/blur/thrift/util/CreateInsertQueryRepeating.java ---------------------------------------------------------------------- diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/util/CreateInsertQueryRepeating.java b/blur-thrift/src/main/java/org/apache/blur/thrift/util/CreateInsertQueryRepeating.java index 88cf16d..afcc351 100644 --- a/blur-thrift/src/main/java/org/apache/blur/thrift/util/CreateInsertQueryRepeating.java +++ b/blur-thrift/src/main/java/org/apache/blur/thrift/util/CreateInsertQueryRepeating.java @@ -160,7 +160,6 @@ public class CreateInsertQueryRepeating { private void testJoin() throws BlurException, TException { RowMutation mutation = new RowMutation(); mutation.table = table; - mutation.waitToBeVisible = true; mutation.rowId = "row1"; mutation.addToRecordMutations(newRecordMutation("cf1", "recordid1", newColumn("col1", "value1"))); mutation.addToRecordMutations(newRecordMutation("cf1", "recordid2", newColumn("col2", "value2"))); @@ -218,7 +217,6 @@ public class CreateInsertQueryRepeating { long buildStart = System.currentTimeMillis(); mutation.clear(); mutation.table = table; - mutation.waitToBeVisible = false; mutation.rowId = UUID.randomUUID().toString(); mutation.addToRecordMutations(newRecordMutation("test", "test-" + i, newColumn("uuidField", UUID.randomUUID().toString()), newColumn("numberField", i + ""), newColumn("fatTextField", randomString(1000)))); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadData.java ---------------------------------------------------------------------- diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadData.java b/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadData.java index ddf6321..477396a 100644 --- a/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadData.java +++ b/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadData.java @@ -44,7 +44,6 @@ public class LoadData { private static List<String> words = new ArrayList<String>(); public static void main(String[] args) throws BlurException, TException, IOException { - final boolean wal = true; final int numberOfColumns = 3; int numberRows = 100000; final int numberRecordsPerRow = 3; @@ -54,11 +53,11 @@ public class LoadData { String connectionString = args[0]; String table = args[1]; Iface client = BlurClient.getClient(connectionString); - runLoad(client, table, wal, numberRows, numberRecordsPerRow, numberOfFamilies, numberOfColumns, numberOfWords, + runLoad(client, table, numberRows, numberRecordsPerRow, numberOfFamilies, numberOfColumns, numberOfWords, batch, new PrintWriter(System.out)); } - public static void runLoad(Iface client, String table, boolean wal, int numberRows, int numberRecordsPerRow, + public static void runLoad(Iface client, String table, int numberRows, int numberRecordsPerRow, int numberOfFamilies, int numberOfColumns, int numberOfWords, int batch, PrintWriter out) throws IOException, BlurException, TException { loadWords(); @@ -85,7 +84,6 @@ public class LoadData { mutation.setTable(table); String rowId = getRowId(); mutation.setRowId(rowId); - mutation.setWal(wal); mutation.setRowMutationType(RowMutationType.REPLACE_ROW); for (int j = 0; j < numberRecordsPerRow; j++) { mutation.addToRecordMutations(getRecordMutation(numberOfColumns, numberOfFamilies, numberOfWords)); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataAsync.java ---------------------------------------------------------------------- diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataAsync.java b/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataAsync.java index 5d1ad74..facdd75 100644 --- a/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataAsync.java +++ b/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataAsync.java @@ -28,6 +28,8 @@ import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thirdparty.thrift_0_9_0.async.AsyncMethodCallback; import org.apache.blur.thrift.AsyncClientPool; import org.apache.blur.thrift.generated.Blur; +import org.apache.blur.thrift.generated.Blur.AsyncClient.mutate_call; +import org.apache.blur.thrift.generated.Blur.AsyncIface; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.Column; import org.apache.blur.thrift.generated.Record; @@ -35,8 +37,6 @@ import org.apache.blur.thrift.generated.RecordMutation; import org.apache.blur.thrift.generated.RecordMutationType; import org.apache.blur.thrift.generated.RowMutation; import org.apache.blur.thrift.generated.RowMutationType; -import org.apache.blur.thrift.generated.Blur.AsyncIface; -import org.apache.blur.thrift.generated.Blur.AsyncClient.mutate_call; public class LoadDataAsync { @@ -46,7 +46,6 @@ public class LoadDataAsync { public static void main(String[] args) throws BlurException, TException, IOException { loadWords(); - final boolean wal = false; final int numberOfColumns = 3; int numberRows = 100000; final int numberRecordsPerRow = 2; @@ -65,7 +64,7 @@ public class LoadDataAsync { System.out.println("Rows indexed [" + i + "] at [" + rate + "/s]"); count = 0; } - client.mutate(getRowMutation(table, wal, numberRecordsPerRow, numberOfColumns, numberOfFamilies, numberOfWords), new AsyncMethodCallback<Blur.AsyncClient.mutate_call>() { + client.mutate(getRowMutation(table, numberRecordsPerRow, numberOfColumns, numberOfFamilies, numberOfWords), new AsyncMethodCallback<Blur.AsyncClient.mutate_call>() { @Override public void onError(Exception exception) { @@ -80,12 +79,11 @@ public class LoadDataAsync { } } - private static RowMutation getRowMutation(String table, boolean wal, int numberRecordsPerRow, int numberOfColumns, int numberOfFamilies, int numberOfWords) { + private static RowMutation getRowMutation(String table, int numberRecordsPerRow, int numberOfColumns, int numberOfFamilies, int numberOfWords) { RowMutation mutation = new RowMutation(); mutation.setTable(table); String rowId = getRowId(); mutation.setRowId(rowId); - mutation.setWal(wal); mutation.setRowMutationType(RowMutationType.REPLACE_ROW); for (int j = 0; j < numberRecordsPerRow; j++) { mutation.addToRecordMutations(getRecordMutation(numberOfColumns, numberOfFamilies, numberOfWords)); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataAsyncContinuously.java ---------------------------------------------------------------------- diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataAsyncContinuously.java b/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataAsyncContinuously.java index 5bb6a7b..b1e2bcb 100644 --- a/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataAsyncContinuously.java +++ b/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataAsyncContinuously.java @@ -93,7 +93,6 @@ public class LoadDataAsyncContinuously { RowMutation mutation = new RowMutation(); mutation.setTable(table); mutation.setRowId(Integer.toString(rowid)); - mutation.setWal(wal); mutation.setRowMutationType(RowMutationType.REPLACE_ROW); for (int j = 0; j < numberRecordsPerRow; j++) { mutation.addToRecordMutations(getRecordMutation(numberOfColumns, numberOfFamilies, numberOfWords)); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataContinuously.java ---------------------------------------------------------------------- diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataContinuously.java b/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataContinuously.java index ec396bc..c09aae8 100644 --- a/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataContinuously.java +++ b/blur-thrift/src/main/java/org/apache/blur/thrift/util/LoadDataContinuously.java @@ -96,7 +96,6 @@ public class LoadDataContinuously { mutation.setTable(table); String rowId = getRowId(); mutation.setRowId(rowId); - mutation.setWal(wal); mutation.setRowMutationType(RowMutationType.REPLACE_ROW); for (int j = 0; j < numberRecordsPerRow; j++) { mutation.addToRecordMutations(getRecordMutation(numberOfColumns, numberOfFamilies, numberOfWords)); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/7315e127/blur-thrift/src/main/java/org/apache/blur/thrift/util/RapidlyCreateAndDeleteTables.java ---------------------------------------------------------------------- diff --git a/blur-thrift/src/main/java/org/apache/blur/thrift/util/RapidlyCreateAndDeleteTables.java b/blur-thrift/src/main/java/org/apache/blur/thrift/util/RapidlyCreateAndDeleteTables.java index 8cc3373..4650c36 100644 --- a/blur-thrift/src/main/java/org/apache/blur/thrift/util/RapidlyCreateAndDeleteTables.java +++ b/blur-thrift/src/main/java/org/apache/blur/thrift/util/RapidlyCreateAndDeleteTables.java @@ -63,7 +63,6 @@ public class RapidlyCreateAndDeleteTables { private static void loadTable(Iface client, String tableName) throws BlurException, TException { RowMutation mutation = new RowMutation(); mutation.table = tableName; - mutation.waitToBeVisible = true; mutation.rowId = "test"; mutation.addToRecordMutations(newRecordMutation("test", "test", newColumn("test", "test"))); mutation.rowMutationType = RowMutationType.REPLACE_ROW;
