Simple BlurCluster test works with a single shard.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/cb538bad Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/cb538bad Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/cb538bad Branch: refs/heads/0.2-dev Commit: cb538bad371c58fdb16d46e94f7b7d5cda16fb9a Parents: aee531c Author: Aaron McCurry <[email protected]> Authored: Thu Nov 22 11:00:00 2012 -0500 Committer: Aaron McCurry <[email protected]> Committed: Thu Nov 22 11:00:00 2012 -0500 ---------------------------------------------------------------------- .../blur/manager/clusterstatus/ClusterStatus.java | 2 + .../clusterstatus/ZookeeperClusterStatus.java | 9 +- .../org/apache/blur/thrift/BlurShardServer.java | 6 +- .../java/org/apache/blur/thrift/TableAdmin.java | 48 +++++----- .../org/apache/blur/thrift/lucene/Convert.java | 33 +++++- .../main/java/org/apache/blur/utils/BlurUtil.java | 2 +- .../src/test/java/org/apache/blur/MiniCluster.java | 51 +++------- .../clusterstatus/ZookeeperClusterStatusTest.java | 77 ++------------ .../org/apache/blur/thrift/BlurClusterTest.java | 63 ++++++++----- .../blur/lucene/serializer/QueryWritableTest.java | 38 +++++++ .../java/org/apache/blur/thrift/Connection.java | 11 +- 11 files changed, 172 insertions(+), 168 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cb538bad/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ClusterStatus.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ClusterStatus.java b/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ClusterStatus.java index b5d48a0..004522a 100644 --- a/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ClusterStatus.java +++ b/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ClusterStatus.java @@ -52,5 +52,7 @@ public abstract class ClusterStatus { public abstract void removeTable(String table, boolean deleteIndexFiles); public abstract boolean isOpen(); + + public abstract void close(); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cb538bad/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java b/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java index 6585e51..cf78777 100644 --- a/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java +++ b/src/blur-core/src/main/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatus.java @@ -29,7 +29,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import org.apache.blur.analysis.BlurAnalyzer; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.thrift.generated.TableDescriptor; @@ -364,6 +363,7 @@ public class ZookeeperClusterStatus extends ClusterStatus { } } + @Override public void close() { if (_running.get()) { _running.set(false); @@ -433,10 +433,9 @@ public class ZookeeperClusterStatus extends ClusterStatus { long s = System.nanoTime(); try { checkIfOpen(); - String table = BlurUtil.nullCheck(tableDescriptor.getName(), "tableDescriptor.name cannot be null."); - BlurAnalyzer analyzer = new BlurAnalyzer(BlurUtil.nullCheck(tableDescriptor.getAnalyzer(), "tableDescriptor.analyzerDefinition cannot be null.")); - String uri = BlurUtil.nullCheck(tableDescriptor.getStoragePath(), "tableDescriptor.tableUri cannot be null."); - int shardCount = BlurUtil.zeroCheck(tableDescriptor.shardCount, "tableDescriptor.shardCount cannot be less than 1"); + String table = BlurUtil.nullCheck(tableDescriptor.getName(), "Name cannot be null."); + String uri = BlurUtil.nullCheck(tableDescriptor.getStoragePath(), "Storage path cannot be null."); + int shardCount = BlurUtil.zeroCheck(tableDescriptor.shardCount, "ShardCount cannot be less than 1"); String blurTablePath = ZookeeperPathConstants.getTablePath(getClusterName(), table); if (_zk.exists(blurTablePath, false) != null) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cb538bad/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java index 1c9f8ef..026530a 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java @@ -203,7 +203,7 @@ public class BlurShardServer extends TableAdmin implements Iface { Collection<Entry<Integer, IndexSearcher>> searchersToSearch = getSearchers(shardIndexes, searchers); List<Future<TopFieldDocs>> futures = new ArrayList<Future<TopFieldDocs>>(searchersToSearch.size()); - Query query = Convert.toLucene(queryArgs); + Query query = Convert.toLuceneQuery(queryArgs.query); Filter filter = Convert.toLuceneFilter(queryArgs); Sort sort = Convert.toLuceneSort(queryArgs); ScoreDoc after = Convert.toLucene(queryArgs.getAfter()); @@ -355,6 +355,10 @@ public class BlurShardServer extends TableAdmin implements Iface { boolean writeAheadLog = options.isWriteAheadLog(); try { BlurIndex index = getIndex(table, shardIndex); + if (index == null) { + System.out.println("Needs to be routed to the correct server [" + shardIndex + "]."); + return; + } index.addDocuments(waitToBeVisible, writeAheadLog, documents); } catch (Throwable t) { LOG.error("Unknown error", t); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cb538bad/src/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java b/src/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java index 5bbf9a7..77f0eda 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java @@ -57,7 +57,7 @@ public abstract class TableAdmin implements Iface { throw new BException(e.getMessage(), e); } if (tableDescriptor.isEnabled()) { - enableTable(tableDescriptor.name); + enableTable(tableDescriptor.getName()); } } @@ -122,28 +122,30 @@ public abstract class TableAdmin implements Iface { } private void waitForTheTableToEngage(String table) throws BlurException, TException { - TableDescriptor tableDescriptor = _clusterStatus.getTableDescriptor(false, table); - int shardCount = tableDescriptor.shardCount; - LOG.info("Waiting for shards to engage on table [" + table + "]"); - while (true) { - try { - Thread.sleep(3000); - } catch (InterruptedException e) { - LOG.error("Unknown error while engaging table [" + table + "]", e); - throw new BException("Unknown error while engaging table [" + table + "]", e); - } - try { - Map<String, String> shardServerLayout = getLayout();//shardServerLayout(table); - LOG.info("Shards [" + shardServerLayout.size() + "/" + shardCount + "] of table [" + table + "] engaged"); - if (shardServerLayout.size() == shardCount) { - return; - } - } catch (BlurException e) { - LOG.info("Stilling waiting", e); - } catch (TException e) { - LOG.info("Stilling waiting", e); - } - } + LOG.info("IMPLEMENT - Waiting for shards to engage on table [" + table + "]"); + +// TableDescriptor tableDescriptor = _clusterStatus.getTableDescriptor(false, table); +// int shardCount = tableDescriptor.shardCount; +// LOG.info("Waiting for shards to engage on table [" + table + "]"); +// while (true) { +// try { +// Thread.sleep(3000); +// } catch (InterruptedException e) { +// LOG.error("Unknown error while engaging table [" + table + "]", e); +// throw new BException("Unknown error while engaging table [" + table + "]", e); +// } +// try { +// Map<String, String> shardServerLayout = getLayout();//shardServerLayout(table); +// LOG.info("Shards [" + shardServerLayout.size() + "/" + shardCount + "] of table [" + table + "] engaged"); +// if (shardServerLayout.size() == shardCount) { +// return; +// } +// } catch (BlurException e) { +// LOG.info("Stilling waiting", e); +// } catch (TException e) { +// LOG.info("Stilling waiting", e); +// } +// } } private Map<String, String> getLayout() throws BlurException, TException { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cb538bad/src/blur-core/src/main/java/org/apache/blur/thrift/lucene/Convert.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/lucene/Convert.java b/src/blur-core/src/main/java/org/apache/blur/thrift/lucene/Convert.java index 38ea977..2a3ce3e 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/lucene/Convert.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/lucene/Convert.java @@ -17,6 +17,7 @@ import org.apache.blur.thrift.generated.TYPE; import org.apache.blur.thrift.generated.TopFieldDocs; import org.apache.blur.utils.BlurUtil; import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; import org.apache.lucene.document.Document; import org.apache.lucene.document.DoubleField; import org.apache.lucene.document.Field.Store; @@ -287,18 +288,40 @@ public class Convert { return result; } + public static Query toLuceneQuery(ByteBuffer byteBuffer) throws IOException { + DataInputBuffer in = new DataInputBuffer(); + in.reset(byteBuffer.array(),byteBuffer.arrayOffset() + byteBuffer.position() , byteBuffer.remaining()); + QueryWritable qw = new QueryWritable(); + qw.readFields(in); + return qw.getQuery(); + } + public static Query toLucene(QueryArgs queryArgs) throws IOException { - return toLuceneQuery(queryArgs.query); + return toLuceneQuery(queryArgs.getQuery()); } - public static Query toLuceneQuery(ByteBuffer buf) throws IOException { + public static Query toLuceneQuery(byte[] bs) throws IOException { DataInputBuffer in = new DataInputBuffer(); - in.reset(buf.array(), buf.arrayOffset(), buf.limit()); + in.reset(bs, bs.length); QueryWritable qw = new QueryWritable(); qw.readFields(in); return qw.getQuery(); } + public static byte[] toBytes(Query query) throws IOException { + DataOutputBuffer out = new DataOutputBuffer(); + QueryWritable qw = new QueryWritable(); + qw.setQuery(query); + qw.write(out); + out.close(); + + byte[] data = out.getData(); + int length = out.getLength(); + byte[] buf = new byte[length]; + System.arraycopy(data, 0, buf, 0, length); + return buf; + } + public static Filter toLuceneFilter(QueryArgs queryArgs) { byte[] filter = queryArgs.getFilter(); return null; @@ -365,7 +388,7 @@ public class Convert { return null; } ScoreDoc result = new ScoreDoc(); - result.setDocLocation(BlurUtil.getDocLocation(scoreDoc.shardIndex,scoreDoc.doc)); + result.setDocLocation(BlurUtil.getDocLocation(scoreDoc.shardIndex, scoreDoc.doc)); result.setScore(scoreDoc.score); return result; } @@ -375,7 +398,7 @@ public class Convert { return null; } ScoreDoc result = new ScoreDoc(); - result.setDocLocation(BlurUtil.getDocLocation(fieldDoc.shardIndex,fieldDoc.doc)); + result.setDocLocation(BlurUtil.getDocLocation(fieldDoc.shardIndex, fieldDoc.doc)); result.setFields(convert(fieldDoc.fields)); result.setScore(fieldDoc.score); return result; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cb538bad/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java index 1fd2ff3..7ba2eed 100644 --- a/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java +++ b/src/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java @@ -441,7 +441,7 @@ public class BlurUtil { } public static void write(byte[] data, TBase<?, ?> base) { - nullCheck(null, "Data cannot be null."); + nullCheck(data, "Data cannot be null."); TMemoryBuffer trans = new TMemoryBuffer(1024); TJSONProtocol protocol = new TJSONProtocol(trans); try { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cb538bad/src/blur-core/src/test/java/org/apache/blur/MiniCluster.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/MiniCluster.java b/src/blur-core/src/test/java/org/apache/blur/MiniCluster.java index 6f19ce5..f649f58 100644 --- a/src/blur-core/src/test/java/org/apache/blur/MiniCluster.java +++ b/src/blur-core/src/test/java/org/apache/blur/MiniCluster.java @@ -17,7 +17,6 @@ package org.apache.blur; * limitations under the License. */ -import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_BIND_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_GUI_CONTROLLER_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_GUI_SHARD_PORT; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_BIND_PORT; @@ -46,7 +45,6 @@ import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.thrift.BlurClientManager; import org.apache.blur.thrift.Connection; -import org.apache.blur.thrift.ThriftBlurControllerServer; import org.apache.blur.thrift.ThriftBlurShardServer; import org.apache.blur.thrift.ThriftServer; import org.apache.blur.thrift.generated.Blur.Client; @@ -74,13 +72,12 @@ public abstract class MiniCluster { private static ZooKeeperServerMainEmbedded zooKeeperServerMain; private static List<ThriftServer> controllers = new ArrayList<ThriftServer>(); private static List<ThriftServer> shards = new ArrayList<ThriftServer>(); - private static String controllerConnectionStr; private static int zkPort; + private static String connectionStr; - public static void startBlurCluster(String name, int controllerCount, int shardCount) { + public static void startBlurCluster(String name, int shardCount) { startDfs(name); startZooKeeper(name); - startControllers(controllerCount); startShards(shardCount); } @@ -91,10 +88,6 @@ public abstract class MiniCluster { shutdownDfs(); } - public static String getControllerConnectionStr() { - return controllerConnectionStr; - } - public static void stopControllers() { for (ThriftServer s : controllers) { s.close(); @@ -107,14 +100,8 @@ public abstract class MiniCluster { } } - public static void startControllers(int num) { - BlurConfiguration configuration = getBlurConfiguration(); - startControllers(configuration, num); - } - private static BlurConfiguration getBlurConfiguration(BlurConfiguration overrides) { BlurConfiguration conf = getBlurConfiguration(); - for (Map.Entry<String, String> over : overrides.getProperties().entrySet()) { conf.set(over.getKey().toString(), over.getValue().toString()); } @@ -139,28 +126,6 @@ public abstract class MiniCluster { return configuration; } - public static void startControllers(BlurConfiguration configuration, int num) { - StringBuilder builder = new StringBuilder(); - BlurConfiguration localConf = getBlurConfiguration(configuration); - int controllerPort = localConf.getInt(BLUR_CONTROLLER_BIND_PORT, 40010); - for (int i = 0; i < num; i++) { - try { - ThriftServer server = ThriftBlurControllerServer.createServer(i, localConf); - controllers.add(server); - Connection connection = new Connection("localhost", controllerPort + i); - if (builder.length() != 0) { - builder.append(','); - } - builder.append(connection.getConnectionStr()); - startServer(server, connection); - } catch (Exception e) { - LOG.error(e); - throw new RuntimeException(e); - } - } - controllerConnectionStr = builder.toString(); - } - public static void startShards(int num) { BlurConfiguration configuration = getBlurConfiguration(); startShards(configuration, num); @@ -179,6 +144,7 @@ public abstract class MiniCluster { } })); } + connectionStr = null; int shardPort = localConf.getInt(BLUR_SHARD_BIND_PORT, 40020); for (int i = 0; i < num; i++) { try { @@ -186,6 +152,11 @@ public abstract class MiniCluster { shards.add(server); Connection connection = new Connection("localhost", shardPort + i); startServer(server, connection); + if (connectionStr == null) { + connectionStr = connection.toString(); + } else { + connectionStr += "," + connection.toString(); + } } catch (Exception e) { LOG.error(e); throw new RuntimeException(e); @@ -349,7 +320,7 @@ public abstract class MiniCluster { File path = new File(TMPDIR, name); System.setProperty("test.build.data", path.getAbsolutePath()); String permission = System.getProperty("dfs.datanode.data.dir.perm"); - if(StringUtils.isNotBlank(permission)){ + if (StringUtils.isNotBlank(permission)) { conf.set("dfs.datanode.data.dir.perm", permission); } try { @@ -432,4 +403,8 @@ public abstract class MiniCluster { file.delete(); } + public static String getConnectionStr() { + return connectionStr; + } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cb538bad/src/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java b/src/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java index 70ec164..c09164a 100644 --- a/src/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java +++ b/src/blur-core/src/test/java/org/apache/blur/manager/clusterstatus/ZookeeperClusterStatusTest.java @@ -19,14 +19,12 @@ package org.apache.blur.manager.clusterstatus; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; import java.util.Arrays; -import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.blur.MiniCluster; @@ -50,7 +48,6 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; - public class ZookeeperClusterStatusTest { private static final String TEST = "test"; @@ -88,7 +85,7 @@ public class ZookeeperClusterStatusTest { } }); BlurUtil.setupZookeeper(zooKeeper, DEFAULT); - clusterStatus = new ZookeeperClusterStatus(zooKeeper); + clusterStatus = new ZookeeperClusterStatus(DEFAULT, zooKeeper); } @After @@ -96,22 +93,20 @@ public class ZookeeperClusterStatusTest { clusterStatus.close(); zooKeeper.close(); } - + @Test - public void testGetClusterList() { - LOG.warn("testGetClusterList"); - List<String> clusterList = clusterStatus.getClusterList(false); - assertEquals(Arrays.asList(DEFAULT), clusterList); + public void createTable() throws KeeperException, InterruptedException { + createTable(TEST, true); } @Test public void testSafeModeNotSet() throws KeeperException, InterruptedException { LOG.warn("testSafeModeNotSet"); - assertFalse(clusterStatus.isInSafeMode(false, DEFAULT)); + assertFalse(clusterStatus.isInSafeMode(false)); new WaitForAnswerToBeCorrect(20L) { @Override public Object run() { - return clusterStatus.isInSafeMode(true, DEFAULT); + return clusterStatus.isInSafeMode(true); } }.test(false); } @@ -120,11 +115,11 @@ public class ZookeeperClusterStatusTest { public void testSafeModeSetInPast() throws KeeperException, InterruptedException { LOG.warn("testSafeModeSetInPast"); setSafeModeInPast(); - assertFalse(clusterStatus.isInSafeMode(false, DEFAULT)); + assertFalse(clusterStatus.isInSafeMode(false)); new WaitForAnswerToBeCorrect(20L) { @Override public Object run() { - return clusterStatus.isInSafeMode(true, DEFAULT); + return clusterStatus.isInSafeMode(true); } }.test(false); } @@ -133,70 +128,20 @@ public class ZookeeperClusterStatusTest { public void testSafeModeSetInFuture() throws KeeperException, InterruptedException { LOG.warn("testSafeModeSetInFuture"); setSafeModeInFuture(); - assertTrue(clusterStatus.isInSafeMode(false, DEFAULT)); + assertTrue(clusterStatus.isInSafeMode(false)); new WaitForAnswerToBeCorrect(20L) { @Override public Object run() { - return clusterStatus.isInSafeMode(true, DEFAULT); + return clusterStatus.isInSafeMode(true); } }.test(true); } @Test - public void testGetClusterNoTable() { - LOG.warn("testGetCluster"); - assertNull(clusterStatus.getCluster(false, TEST)); - assertNull(clusterStatus.getCluster(true, TEST)); - } - - @Test - public void testGetClusterTable() throws KeeperException, InterruptedException { - LOG.warn("testGetCluster"); - createTable(TEST); - assertEquals(DEFAULT, clusterStatus.getCluster(false, TEST)); - new WaitForAnswerToBeCorrect(20L) { - @Override - public Object run() { - return clusterStatus.getCluster(true, TEST); - } - }.test(DEFAULT); - } - - @Test public void testGetTableList() { assertEquals(Arrays.asList(TEST), clusterStatus.getTableList(false)); } - @Test - public void testIsEnabledNoTable() { - assertFalse(clusterStatus.isEnabled(false, DEFAULT, "notable")); - assertFalse(clusterStatus.isEnabled(true, DEFAULT, "notable")); - } - - @Test - public void testIsEnabledDisabledTable() throws KeeperException, InterruptedException { - createTable("disabledtable", false); - assertFalse(clusterStatus.isEnabled(false, DEFAULT, "disabledtable")); - assertFalse(clusterStatus.isEnabled(true, DEFAULT, "disabledtable")); - } - - @Test - public void testIsEnabledEnabledTable() throws KeeperException, InterruptedException { - createTable("enabledtable", true); - assertTrue(clusterStatus.isEnabled(false, DEFAULT, "enabledtable")); - - new WaitForAnswerToBeCorrect(20L) { - @Override - public Object run() { - return clusterStatus.isEnabled(true, DEFAULT, "enabledtable"); - } - }.test(true); - } - - private void createTable(String name) throws KeeperException, InterruptedException { - createTable(name, true); - } - private void createTable(String name, boolean enabled) throws KeeperException, InterruptedException { TableDescriptor tableDescriptor = new TableDescriptor(); tableDescriptor.setName(name); @@ -205,7 +150,7 @@ public class ZookeeperClusterStatusTest { tableDescriptor.setEnabled(enabled); clusterStatus.createTable(tableDescriptor); if (enabled) { - clusterStatus.enableTable(tableDescriptor.getClusterName(), name); + clusterStatus.enableTable(name); } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cb538bad/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java index 3439412..39b428e 100644 --- a/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java +++ b/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java @@ -21,20 +21,33 @@ import static org.junit.Assert.assertEquals; import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.UUID; import org.apache.blur.MiniCluster; import org.apache.blur.thrift.generated.Blur; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.thrift.generated.Document; +import org.apache.blur.thrift.generated.Field; +import org.apache.blur.thrift.generated.MutateOptions; +import org.apache.blur.thrift.generated.QueryArgs; +import org.apache.blur.thrift.generated.Session; +import org.apache.blur.thrift.generated.TYPE; import org.apache.blur.thrift.generated.TableDescriptor; +import org.apache.blur.thrift.generated.TopFieldDocs; +import org.apache.blur.thrift.lucene.Convert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.TermQuery; import org.apache.thrift.TException; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -64,7 +77,7 @@ public class BlurClusterTest { System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum); testDirectory.delete(); - MiniCluster.startBlurCluster("cluster", 2, 3); + MiniCluster.startBlurCluster("cluster", 1); } @AfterClass @@ -78,34 +91,38 @@ public class BlurClusterTest { TableDescriptor tableDescriptor = new TableDescriptor(); tableDescriptor.setName("test"); tableDescriptor.setShardCount(5); - tableDescriptor.setTableUri(MiniCluster.getFileSystemUri().toString() + "/blur/test"); + tableDescriptor.setStoragePath(MiniCluster.getFileSystemUri().toString() + "/blur/test"); client.createTable(tableDescriptor); List<String> tableList = client.tableList(); assertEquals(Arrays.asList("test"), tableList); } private Iface getClient() { - return BlurClient.getClient(MiniCluster.getControllerConnectionStr()); + return BlurClient.getClient(MiniCluster.getConnectionStr()); } -// @Test -// public void testLoadTable() throws BlurException, TException, InterruptedException { -// Iface client = getClient(); -// int length = 100; -// List<RowMutation> mutations = new ArrayList<RowMutation>(); -// for (int i = 0; i < length; i++) { -// String rowId = UUID.randomUUID().toString(); -// RecordMutation mutation = BlurUtil.newRecordMutation("test", rowId, BlurUtil.newColumn("test", "value")); -// RowMutation rowMutation = BlurUtil.newRowMutation("test", rowId, mutation); -// rowMutation.setWaitToBeVisible(true); -// mutations.add(rowMutation); -// } -// client.mutateBatch(mutations); -// BlurQuery blurQuery = new BlurQuery(); -// SimpleQuery simpleQuery = new SimpleQuery(); -// simpleQuery.setQueryStr("test.test:value"); -// blurQuery.setSimpleQuery(simpleQuery); -// BlurResults results = client.query("test", blurQuery); -// assertEquals(length, results.getTotalResults()); -// } + @Test + public void testLoadTable() throws BlurException, TException, InterruptedException, IOException { + Iface client = getClient(); + int length = 100; + List<Document> documents = new ArrayList<Document>(); + for (int i = 0; i < length; i++) { + Document doc = new Document(); + doc.addToFields(new Field("id", ByteBuffer.wrap(UUID.randomUUID().toString().getBytes()), TYPE.STRING, 1.0)); + doc.addToFields(new Field("value", ByteBuffer.wrap("test".getBytes()), TYPE.STRING, 1.0)); + documents.add(doc); + } + MutateOptions options = new MutateOptions(); + options.setTable("test"); + options.setWaitToBeVisible(true); + client.addDocuments(options, documents); + + Session session = client.openReadSession("test"); + QueryArgs queryArgs = new QueryArgs(); + Term term = new Term("value", "test"); + TermQuery query = new TermQuery(term); + queryArgs.setQuery(Convert.toBytes(query)); + List<TopFieldDocs> results = client.search(session, queryArgs); + System.out.println(results); + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cb538bad/src/blur-store/src/test/java/org/apache/blur/lucene/serializer/QueryWritableTest.java ---------------------------------------------------------------------- diff --git a/src/blur-store/src/test/java/org/apache/blur/lucene/serializer/QueryWritableTest.java b/src/blur-store/src/test/java/org/apache/blur/lucene/serializer/QueryWritableTest.java new file mode 100644 index 0000000..2b1e87c --- /dev/null +++ b/src/blur-store/src/test/java/org/apache/blur/lucene/serializer/QueryWritableTest.java @@ -0,0 +1,38 @@ +package org.apache.blur.lucene.serializer; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; + +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.junit.Test; + +public class QueryWritableTest { + + @Test + public void testTermQuery() throws IOException { + TermQuery query = new TermQuery(new Term("field","value")); + QueryWritable queryWritable = new QueryWritable(); + queryWritable.setQuery(query); + DataOutputBuffer out = new DataOutputBuffer(); + queryWritable.write(out); + byte[] data = out.getData(); + int length = out.getLength(); + + DataInputBuffer in = new DataInputBuffer(); + in.reset(data, length); + + QueryWritable newQueryWritable = new QueryWritable(); + newQueryWritable.readFields(in); + + Query termQuery = newQueryWritable.getQuery(); + + assertEquals(query,termQuery); + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/cb538bad/src/blur-thrift/src/main/java/org/apache/blur/thrift/Connection.java ---------------------------------------------------------------------- diff --git a/src/blur-thrift/src/main/java/org/apache/blur/thrift/Connection.java b/src/blur-thrift/src/main/java/org/apache/blur/thrift/Connection.java index e9ce24b..a2044f1 100644 --- a/src/blur-thrift/src/main/java/org/apache/blur/thrift/Connection.java +++ b/src/blur-thrift/src/main/java/org/apache/blur/thrift/Connection.java @@ -34,6 +34,7 @@ public class Connection { int indexOfProxyPort = connectionStr.indexOf(':', slashIndex); _proxyHost = connectionStr.substring(slashIndex + 1, indexOfProxyPort); _proxyPort = Integer.parseInt(connectionStr.substring(indexOfProxyPort + 1)); + _proxy = true; } else { _host = connectionStr.substring(0, index); _port = Integer.parseInt(connectionStr.substring(index + 1)); @@ -118,13 +119,11 @@ public class Connection { @Override public String toString() { - return "Connection [_host=" + _host + ", _port=" + _port + ", _proxy=" + _proxy + ", _proxyHost=" + _proxyHost + ", _proxyPort=" + _proxyPort + "]"; - } - - public Object getConnectionStr() { - if (_proxyHost != null) { + if (_proxy) { return _host + ":" + _port + "/" + _proxyHost + ":" + _proxyPort; + } else { + return _host + ":" + _port; } - return _host + ":" + _port; } + }
