Now the update documents call works.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/64943aa0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/64943aa0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/64943aa0 Branch: refs/heads/0.2-dev Commit: 64943aa0bef4cfc93595b47e0277ccbafe571bf9 Parents: 8018b06 Author: Aaron McCurry <[email protected]> Authored: Sat Nov 24 20:46:19 2012 -0500 Committer: Aaron McCurry <[email protected]> Committed: Sat Nov 24 20:46:19 2012 -0500 ---------------------------------------------------------------------- .../org/apache/blur/thrift/BlurShardServer.java | 19 +- .../org/apache/blur/thrift/BlurClusterTest.java | 137 ------ .../blur/thrift/BlurControllerServerTest.java | 334 --------------- .../apache/blur/thrift/BlurShardServerTest.java | 151 ------- .../org/apache/blur/thrift/DoNothingServer.java | 134 ------ .../org/apache/blur/thrift/ITBlurClusterTest.java | 186 ++++++++ 6 files changed, 204 insertions(+), 757 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/64943aa0/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java index 5526f85..088370f 100644 --- a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java +++ b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurShardServer.java @@ -426,6 +426,19 @@ public class BlurShardServer extends TableAdmin implements Iface { } }); } + + private void forwardUpdateDocuments(final MutateOptions options, final List<UpdatePackage> updatePackages) throws BlurException, TException, IOException { + String table = options.getTable(); + int shardIndex = options.getShardIndex(); + Connection connection = getConnection(table, shardIndex); + BlurClientManager.execute(connection, new BlurCommand<Void>() { + @Override + public Void call(Client client) throws BlurException, TException { + client.updateDocuments(options, updatePackages); + return null; + } + }); + } private Connection getConnection(String table, int shardIndex) { String server = _layout.findServer(table, shardIndex, TYPE.WRITABLE); @@ -470,7 +483,11 @@ public class BlurShardServer extends TableAdmin implements Iface { boolean writeAheadLog = options.isWriteAheadLog(); try { BlurIndex index = getIndex(table, shardIndex); - index.updateDocuments(waitToBeVisible, writeAheadLog, updatePackages); + if (index == null) { + forwardUpdateDocuments(options, updatePackages); + } else { + index.updateDocuments(waitToBeVisible, writeAheadLog, updatePackages); + } } catch (Throwable t) { LOG.error("Unknown error", t); throw new BException(t.getMessage(), t); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/64943aa0/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java deleted file mode 100644 index de97d6d..0000000 --- a/src/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java +++ /dev/null @@ -1,137 +0,0 @@ -package org.apache.blur.thrift; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.UUID; - -import org.apache.blur.MiniCluster; -import org.apache.blur.thrift.generated.Blur; -import org.apache.blur.thrift.generated.Blur.Iface; -import org.apache.blur.thrift.generated.BlurException; -import org.apache.blur.thrift.generated.Document; -import org.apache.blur.thrift.generated.Field; -import org.apache.blur.thrift.generated.MutateOptions; -import org.apache.blur.thrift.generated.QueryArgs; -import org.apache.blur.thrift.generated.Session; -import org.apache.blur.thrift.generated.TYPE; -import org.apache.blur.thrift.generated.TableDescriptor; -import org.apache.blur.thrift.generated.TopFieldDocs; -import org.apache.blur.thrift.lucene.Convert; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.TermQuery; -import org.apache.thrift.TException; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public class BlurClusterTest { - - private static final int SHARD_COUNT = 5; - private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "/tmp")); - private static final int SHARD_SERVERS = 2; - - @BeforeClass - public static void startCluster() throws IOException { - LocalFileSystem localFS = FileSystem.getLocal(new Configuration()); - File testDirectory = new File(TMPDIR, "blur-cluster-test"); - testDirectory.mkdirs(); - - Path directory = new Path(testDirectory.getPath()); - FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission(); - FsAction userAction = dirPermissions.getUserAction(); - FsAction groupAction = dirPermissions.getGroupAction(); - FsAction otherAction = dirPermissions.getOtherAction(); - - StringBuilder builder = new StringBuilder(); - builder.append(userAction.ordinal()); - builder.append(groupAction.ordinal()); - builder.append(otherAction.ordinal()); - String dirPermissionNum = builder.toString(); - System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum); - testDirectory.delete(); - - MiniCluster.startBlurCluster("cluster", SHARD_SERVERS); - } - - @AfterClass - public static void shutdownCluster() { - MiniCluster.shutdownBlurCluster(); - } - - @Test - public void testCreateTable() throws BlurException, TException, IOException { - Blur.Iface client = getClient(); - TableDescriptor tableDescriptor = new TableDescriptor(); - tableDescriptor.setName("test"); - tableDescriptor.setShardCount(SHARD_COUNT); - tableDescriptor.setStoragePath(MiniCluster.getFileSystemUri().toString() + "/blur/test"); - client.createTable(tableDescriptor); - List<String> tableList = client.tableList(); - assertEquals(Arrays.asList("test"), tableList); - } - - private Iface getClient() { - return BlurClient.getClient(MiniCluster.getConnectionStr(), 0, 0, 0); - } - - @Test - public void testLoadTable() throws BlurException, TException, InterruptedException, IOException { - Iface client = getClient(); - for (int s = 0; s < SHARD_COUNT; s++) { - int length = 100; - List<Document> documents = new ArrayList<Document>(); - for (int i = 0; i < length; i++) { - Document doc = new Document(); - doc.addToFields(new Field("id", ByteBuffer.wrap(UUID.randomUUID().toString().getBytes()), TYPE.STRING, 1.0)); - doc.addToFields(new Field("value", ByteBuffer.wrap("test".getBytes()), TYPE.STRING, 1.0)); - documents.add(doc); - } - MutateOptions options = new MutateOptions(); - options.setTable("test"); - options.setWaitToBeVisible(true); - options.setShardIndex(s); - client.addDocuments(options, documents); - } - - Session session = client.openReadSession("test"); - QueryArgs queryArgs = new QueryArgs(); - Term term = new Term("value", "test"); - TermQuery query = new TermQuery(term); - queryArgs.setQuery(Convert.toBytes(query)); - List<TopFieldDocs> results = client.search(session, queryArgs); - long totalHits = 0; - for (TopFieldDocs fieldDocs : results) { - totalHits += fieldDocs.getTotalHits(); - } - assertEquals(SHARD_COUNT * 100, totalHits); - } -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/64943aa0/src/blur-core/src/test/java/org/apache/blur/thrift/BlurControllerServerTest.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/BlurControllerServerTest.java b/src/blur-core/src/test/java/org/apache/blur/thrift/BlurControllerServerTest.java deleted file mode 100644 index dd2f777..0000000 --- a/src/blur-core/src/test/java/org/apache/blur/thrift/BlurControllerServerTest.java +++ /dev/null @@ -1,334 +0,0 @@ -package org.apache.blur.thrift; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.junit.Test; - -public class BlurControllerServerTest { - - @Test - public void testNothing() { - // do nothing - } - - // private static final String TABLE = "test"; - // private static Map<String, Iface> shardServers = new HashMap<String, - // Iface>(); - // private static BlurControllerServer server; - // private static Thread daemonService; - // - // @BeforeClass - // public static void zkServerStartup() throws InterruptedException, - // KeeperException, IOException, BlurException, TException { - // rm(new File(TMPDIR, "../zk-tmp")); - // daemonService = new Thread(new Runnable() { - // @Override - // public void run() { - // QuorumPeerMain.main(new - // String[]{"./src/test/resources/com/nearinfinity/blur/thrift/zoo.cfg"}); - // } - // }); - // daemonService.start(); - // Thread.sleep(1000); - // - // ZooKeeper zookeeper = new ZooKeeper("127.0.0.1:10101", 30000, new Watcher() - // { - // @Override - // public void process(WatchedEvent event) { - // - // } - // }); - // - // addShardServer("shard-00000000"); - // addShardServer("shard-00000001"); - // addShardServer("shard-00000002"); - // BlurQueryChecker queryChecker = new BlurQueryChecker(new - // BlurConfiguration()); - // server = new BlurControllerServer(); - // server.setClient(getClient()); - // server.setClusterStatus(new ZookeeperClusterStatus(zookeeper)); - // server.setQueryChecker(queryChecker); - // server.setZookeeper(zookeeper); - // server.init(); - // - // File file = new File(TMPDIR, "./tmp-data/test"); - // rm(file); - // - // TableDescriptor tableDescriptor = new TableDescriptor(); - // tableDescriptor.name = "test"; - // tableDescriptor.shardCount = 3; - // tableDescriptor.tableUri = file.toURI().toString(); - // tableDescriptor.analyzerDefinition = new AnalyzerDefinition(); - // server.createTable(tableDescriptor); - // } - // - // @AfterClass - // public static void zkServerShutdown() { - // daemonService.interrupt(); - // server.close(); - // } - // - // private static void rm(File file) { - // if (file.isDirectory()) { - // for (File f : file.listFiles()) { - // rm(f); - // } - // } - // file.delete(); - // } - // - // @Test - // public void testQuery() throws BlurException, TException { - // BlurQuery blurQuery = new BlurQuery(); - // blurQuery.maxQueryTime = TimeUnit.SECONDS.toMillis(5); - // blurQuery.minimumNumberOfResults = Long.MAX_VALUE; - // BlurResults results = server.query(TABLE, blurQuery); - // assertNotNull(results); - // } - // - // @Test - // public void testQueryWithFacets() throws BlurException, TException { - // BlurQuery blurQuery = new BlurQuery(); - // blurQuery.maxQueryTime = TimeUnit.SECONDS.toMillis(5); - // blurQuery.minimumNumberOfResults = Long.MAX_VALUE; - // blurQuery.facets = new ArrayList<Facet>(); - // blurQuery.facets.add(new Facet()); - // blurQuery.facets.add(new Facet()); - // BlurResults results = server.query(TABLE, blurQuery); - // assertNotNull(results); - // assertNotNull(results.facetCounts); - // for (int i = 0; i < results.facetCounts.size(); i++) { - // long count = results.facetCounts.get(i); - // assertEquals(shardServers.size() * (i + 1), count); - // } - // } - // - // @Test - // public void testRecordFrequency() throws BlurException, TException { - // long recordFrequency = server.recordFrequency(TABLE, "cf", "cn", "value"); - // assertEquals(shardServers.size(), recordFrequency); - // } - // - // @Test - // public void testMutate() throws BlurException, TException { - // RowMutation mutation = new RowMutation(); - // mutation.setRowId("1234"); - // RecordMutation recMut = new RecordMutation(); - // Record record = new Record(); - // record.setFamily("test"); - // record.setRecordId("5678"); - // record.addToColumns(new Column("name", "value")); - // mutation.addToRecordMutations(recMut); - // mutation.table = TABLE; - // server.mutate(mutation); - // - // Selector selector = new Selector(); - // selector.rowId = "1234"; - // - // FetchResult fetchRow = server.fetchRow(TABLE, selector); - // assertNotNull(fetchRow.rowResult); - // } - // - // private static BlurClient getClient() { - // BlurClientEmbedded blurClientEmbedded = new BlurClientEmbedded(); - // for (String node : shardServers.keySet()) { - // blurClientEmbedded.putNode(node, shardServers.get(node)); - // } - // return blurClientEmbedded; - // } - // - // private static Iface getShardServer(final String node) { - // return new Iface() { - // - // private Map<String, Map<String, Row>> rows = new HashMap<String, - // Map<String, Row>>(); - // - // @Override - // public List<String> terms(String arg0, String arg1, String arg2, String - // arg3, short arg4) throws BlurException, TException { - // throw new RuntimeException("no impl"); - // } - // - // @Override - // public List<String> tableList() throws BlurException, TException { - // List<String> table = new ArrayList<String>(); - // table.add(TABLE); - // return table; - // } - // - // @Override - // public List<String> shardServerList(String cluster) throws BlurException, - // TException { - // throw new RuntimeException("no impl"); - // } - // - // @Override - // public Map<String, String> shardServerLayout(String table) throws - // BlurException, TException { - // Map<String, String> layout = new HashMap<String, String>(); - // layout.put(node, node); - // return layout; - // } - // - // @Override - // public BlurResults query(String table, BlurQuery query) throws - // BlurException, TException { - // BlurResults results = new BlurResults(); - // results.putToShardInfo(node, 0); - // results.setFacetCounts(getFacetCounts(query)); - // return results; - // } - // - // private List<Long> getFacetCounts(BlurQuery query) { - // if (query.facets != null) { - // int size = query.facets.size(); - // List<Long> results = new ArrayList<Long>(); - // for (int i = 0; i < size; i++) { - // results.add(i + 1L); - // } - // return results; - // } - // return null; - // } - // - // @Override - // public Schema schema(String arg0) throws BlurException, TException { - // throw new RuntimeException("no impl"); - // } - // - // @Override - // public long recordFrequency(String arg0, String arg1, String arg2, String - // arg3) throws BlurException, TException { - // return 1l; - // } - // - // @Override - // public FetchResult fetchRow(String table, Selector selector) throws - // BlurException, TException { - // Map<String, Row> map = rows.get(table); - // Row row = map.get(selector.rowId); - // FetchResult fetchResult = new FetchResult(); - // fetchResult.setRowResult(new FetchRowResult(row)); - // return fetchResult; - // } - // - // @Override - // public TableDescriptor describe(String arg0) throws BlurException, - // TException { - // TableDescriptor descriptor = new TableDescriptor(); - // descriptor.isEnabled = true; - // descriptor.shardCount = 3; - // return descriptor; - // } - // - // @Override - // public List<BlurQueryStatus> currentQueries(String arg0) throws - // BlurException, TException { - // throw new RuntimeException("no impl"); - // } - // - // @Override - // public List<String> controllerServerList() throws BlurException, TException - // { - // throw new RuntimeException("no impl"); - // } - // - // @Override - // public void cancelQuery(String table, long arg0) throws BlurException, - // TException { - // throw new RuntimeException("no impl"); - // } - // - // private Row toRow(RowMutation mutation) { - // Row row = new Row(); - // row.id = mutation.rowId; - // row.records = toRecords(mutation.recordMutations); - // return row; - // } - // - // private List<Record> toRecords(List<RecordMutation> recordMutations) { - // List<Record> records = new ArrayList<Record>(); - // for (RecordMutation mutation : recordMutations) { - // records.add(mutation.record); - // } - // return records; - // } - // - // @Override - // public void createTable(TableDescriptor tableDescriptor) throws - // BlurException, TException { - // throw new RuntimeException("not impl"); - // } - // - // @Override - // public void disableTable(String table) throws BlurException, TException { - // throw new RuntimeException("not impl"); - // } - // - // @Override - // public void enableTable(String table) throws BlurException, TException { - // throw new RuntimeException("not impl"); - // } - // - // @Override - // public void removeTable(String table, boolean deleteIndexFiles) throws - // BlurException, TException { - // throw new RuntimeException("not impl"); - // } - // - // @Override - // public TableStats getTableStats(String table) throws BlurException, - // TException { - // return new TableStats(); - // } - // - // @Override - // public void mutate(RowMutation mutation) throws BlurException, TException { - // String table = mutation.table; - // Map<String, Row> map = rows.get(table); - // if (map == null) { - // map = new HashMap<String, Row>(); - // rows.put(table, map); - // } - // Row row = toRow(mutation); - // map.put(row.id, row); - // } - // - // @Override - // public void mutateBatch(List<RowMutation> mutations) throws BlurException, - // TException { - // for (RowMutation mutation : mutations) { - // MutationHelper.validateMutation(mutation); - // } - // for (RowMutation mutation : mutations) { - // mutate(mutation); - // } - // } - // - // @Override - // public List<String> shardClusterList() throws BlurException, TException { - // throw new RuntimeException("no impl"); - // } - // }; - // } - // - // private static void addShardServer(String node) { - // shardServers.put(node, getShardServer(node)); - // } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/64943aa0/src/blur-core/src/test/java/org/apache/blur/thrift/BlurShardServerTest.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/BlurShardServerTest.java b/src/blur-core/src/test/java/org/apache/blur/thrift/BlurShardServerTest.java deleted file mode 100644 index 7f78ba4..0000000 --- a/src/blur-core/src/test/java/org/apache/blur/thrift/BlurShardServerTest.java +++ /dev/null @@ -1,151 +0,0 @@ -package org.apache.blur.thrift; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.SortedSet; - -import org.apache.blur.BlurConfiguration; -import org.apache.blur.analysis.BlurAnalyzer; -import org.apache.blur.manager.IndexManager; -import org.apache.blur.manager.IndexServer; -import org.apache.blur.manager.writer.BlurIndex; -import org.apache.blur.thrift.generated.BlurException; -import org.apache.hadoop.io.compress.CompressionCodec; -import org.apache.lucene.search.similarities.Similarity; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class BlurShardServerTest { - - private BlurShardServer blurShardServer; - - @Before - public void setUp() throws BlurException, IOException { - IndexServer indexServer = getIndexServer(); - IndexManager indexManager = getIndexManager(); - indexManager.setIndexServer(indexServer); - indexManager.setThreadCount(1); - indexManager.init(); - blurShardServer = new BlurShardServer(); - blurShardServer.setIndexManager(indexManager); - blurShardServer.setIndexServer(indexServer); - blurShardServer.setConfiguration(new BlurConfiguration()); - blurShardServer.init(); - } - - @After - public void tearDown() throws InterruptedException { - blurShardServer.close(); - } - - @Test - public void testNothingYet() { - - } - - private IndexServer getIndexServer() { - return new IndexServer() { - - @Override - public void close() { - - } - - @Override - public BlurAnalyzer getAnalyzer(String table) { - throw new RuntimeException("not impl"); - } - - @Override - public Map<String, BlurIndex> getIndexes(String table) throws IOException { - throw new RuntimeException("not impl"); - } - - @Override - public List<String> getShardList(String table) { - throw new RuntimeException("not impl"); - } - - @Override - public Similarity getSimilarity(String table) { - throw new RuntimeException("not impl"); - } - - @Override - public TABLE_STATUS getTableStatus(String table) { - throw new RuntimeException("not impl"); - } - - @Override - public String getNodeName() { - throw new RuntimeException("no impl"); - } - - @Override - public String getTableUri(String table) { - throw new RuntimeException("no impl"); - } - - @Override - public int getShardCount(String table) { - throw new RuntimeException("no impl"); - } - - @Override - public int getCompressionBlockSize(String table) { - throw new RuntimeException("no impl"); - } - - @Override - public CompressionCodec getCompressionCodec(String table) { - throw new RuntimeException("no impl"); - } - - @Override - public long getRecordCount(String table) throws IOException { - throw new RuntimeException("no impl"); - } - - @Override - public long getRowCount(String table) throws IOException { - throw new RuntimeException("no impl"); - } - - @Override - public long getTableSize(String table) throws IOException { - throw new RuntimeException("no impl"); - } - - @Override - public SortedSet<String> getShardListCurrentServerOnly(String table) throws IOException { - throw new RuntimeException("no impl"); - } - }; - } - - private IndexManager getIndexManager() { - return new IndexManager() { - - }; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/64943aa0/src/blur-core/src/test/java/org/apache/blur/thrift/DoNothingServer.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/DoNothingServer.java b/src/blur-core/src/test/java/org/apache/blur/thrift/DoNothingServer.java deleted file mode 100644 index e9f1a7a..0000000 --- a/src/blur-core/src/test/java/org/apache/blur/thrift/DoNothingServer.java +++ /dev/null @@ -1,134 +0,0 @@ -package org.apache.blur.thrift; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import org.apache.blur.thrift.generated.Blur.Iface; -import org.apache.blur.thrift.generated.BlurException; -import org.apache.blur.thrift.generated.BlurQueryStatus; -import org.apache.blur.thrift.generated.Schema; -import org.apache.blur.thrift.generated.TableDescriptor; -import org.apache.blur.thrift.generated.TableStats; -import org.apache.thrift.TException; - - -public class DoNothingServer implements Iface { - - @Override - public TableDescriptor describe(String table) throws BlurException, TException { - return null; - } - - @Override - public List<String> tableList() throws BlurException, TException { - return Arrays.asList("donothing"); - } - - @Override - public List<String> controllerServerList() throws BlurException, TException { - return null; - } - - @Override - public List<String> shardServerList(String cluster) throws BlurException, TException { - return null; - } - - @Override - public Map<String, String> shardServerLayout(String table) throws BlurException, TException { - return null; - } - - @Override - public void cancelQuery(String table, long providedUuid) throws BlurException, TException { - - } - - @Override - public List<BlurQueryStatus> currentQueries(String arg0) throws BlurException, TException { - return null; - } - - @Override - public Schema schema(String arg0) throws BlurException, TException { - return null; - } - - @Override - public void createTable(TableDescriptor tableDescriptor) throws BlurException, TException { - - } - - @Override - public void disableTable(String table) throws BlurException, TException { - - } - - @Override - public void enableTable(String table) throws BlurException, TException { - - } - - @Override - public void removeTable(String table, boolean deleteIndexFiles) throws BlurException, TException { - - } - - @Override - public List<String> shardClusterList() throws BlurException, TException { - return null; - } - - @Override - public List<String> tableListByCluster(String cluster) throws BlurException, TException { - return null; - } - - @Override - public BlurQueryStatus queryStatusById(String table, long uuid) throws BlurException, TException { - return null; - } - - @Override - public List<Long> queryStatusIdList(String table) throws BlurException, TException { - return null; - } - - @Override - public void optimize(String table, int numberOfSegmentsPerShard) throws BlurException, TException { - - } - - @Override - public boolean isInSafeMode(String cluster) throws BlurException, TException { - return false; - } - - @Override - public TableStats tableStats(String table) throws BlurException, TException { - return null; - } - - @Override - public Map<String, String> configuration() throws BlurException, TException { - return null; - } -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/64943aa0/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java ---------------------------------------------------------------------- diff --git a/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java b/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java new file mode 100644 index 0000000..dd0e4b1 --- /dev/null +++ b/src/blur-core/src/test/java/org/apache/blur/thrift/ITBlurClusterTest.java @@ -0,0 +1,186 @@ +package org.apache.blur.thrift; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +import org.apache.blur.MiniCluster; +import org.apache.blur.thrift.generated.Blur; +import org.apache.blur.thrift.generated.Blur.Iface; +import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.thrift.generated.Document; +import org.apache.blur.thrift.generated.Field; +import org.apache.blur.thrift.generated.MutateOptions; +import org.apache.blur.thrift.generated.QueryArgs; +import org.apache.blur.thrift.generated.Session; +import org.apache.blur.thrift.generated.TYPE; +import org.apache.blur.thrift.generated.TableDescriptor; +import org.apache.blur.thrift.generated.TopFieldDocs; +import org.apache.blur.thrift.generated.UpdatePackage; +import org.apache.blur.thrift.lucene.Convert; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.TermQuery; +import org.apache.thrift.TException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ITBlurClusterTest { + + private static final int SHARD_COUNT = 5; + private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "/tmp")); + private static final int SHARD_SERVERS = 2; + + @BeforeClass + public static void startCluster() throws IOException { + LocalFileSystem localFS = FileSystem.getLocal(new Configuration()); + File testDirectory = new File(TMPDIR, "blur-cluster-test"); + testDirectory.mkdirs(); + + Path directory = new Path(testDirectory.getPath()); + FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission(); + FsAction userAction = dirPermissions.getUserAction(); + FsAction groupAction = dirPermissions.getGroupAction(); + FsAction otherAction = dirPermissions.getOtherAction(); + + StringBuilder builder = new StringBuilder(); + builder.append(userAction.ordinal()); + builder.append(groupAction.ordinal()); + builder.append(otherAction.ordinal()); + String dirPermissionNum = builder.toString(); + System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum); + testDirectory.delete(); + + MiniCluster.startBlurCluster("cluster", SHARD_SERVERS); + } + + @AfterClass + public static void shutdownCluster() { + MiniCluster.shutdownBlurCluster(); + } + + @Test + public void testCreateTable() throws BlurException, TException, IOException { + Blur.Iface client = getClient(); + TableDescriptor tableDescriptor = new TableDescriptor(); + tableDescriptor.setName("test"); + tableDescriptor.setShardCount(SHARD_COUNT); + tableDescriptor.setStoragePath(MiniCluster.getFileSystemUri().toString() + "/blur/test"); + client.createTable(tableDescriptor); + List<String> tableList = client.tableList(); + assertEquals(Arrays.asList("test"), tableList); + } + + private Iface getClient() { + return BlurClient.getClient(MiniCluster.getConnectionStr(), 0, 0, 0); + } + + @Test + public void testAddDocuments() throws BlurException, TException, InterruptedException, IOException { + Iface client = getClient(); + TableDescriptor tableDescriptor = new TableDescriptor(); + tableDescriptor.setName("test-add-table"); + tableDescriptor.setShardCount(SHARD_COUNT); + tableDescriptor.setStoragePath(MiniCluster.getFileSystemUri().toString() + "/blur/test-add-table"); + client.createTable(tableDescriptor); + + for (int s = 0; s < SHARD_COUNT; s++) { + int length = 100; + List<Document> documents = new ArrayList<Document>(); + for (int i = 0; i < length; i++) { + Document doc = new Document(); + doc.addToFields(new Field("id", ByteBuffer.wrap(UUID.randomUUID().toString().getBytes()), TYPE.STRING, 1.0)); + doc.addToFields(new Field("value", ByteBuffer.wrap("test".getBytes()), TYPE.STRING, 1.0)); + documents.add(doc); + } + MutateOptions options = new MutateOptions(); + options.setTable(tableDescriptor.getName()); + options.setWaitToBeVisible(true); + options.setShardIndex(s); + client.addDocuments(options, documents); + } + + Session session = client.openReadSession(tableDescriptor.getName()); + QueryArgs queryArgs = new QueryArgs(); + Term term = new Term("value", "test"); + TermQuery query = new TermQuery(term); + queryArgs.setQuery(Convert.toBytes(query)); + List<TopFieldDocs> results = client.search(session, queryArgs); + long totalHits = 0; + for (TopFieldDocs fieldDocs : results) { + totalHits += fieldDocs.getTotalHits(); + } + assertEquals(SHARD_COUNT * 100, totalHits); + } + + @Test + public void testUpdateDocuments() throws BlurException, TException, InterruptedException, IOException { + Iface client = getClient(); + TableDescriptor tableDescriptor = new TableDescriptor(); + tableDescriptor.setName("test-update-table"); + tableDescriptor.setShardCount(SHARD_COUNT); + tableDescriptor.setStoragePath(MiniCluster.getFileSystemUri().toString() + "/blur/test-update-table"); + client.createTable(tableDescriptor); + + for (int s = 0; s < SHARD_COUNT; s++) { + int length = 100; + List<UpdatePackage> updatePackages = new ArrayList<UpdatePackage>(); + for (int i = 0; i < length; i++) { + UpdatePackage updatePackage = new UpdatePackage(); + Document doc = new Document(); + byte[] uuid = UUID.randomUUID().toString().getBytes(); + doc.addToFields(new Field("id", ByteBuffer.wrap(uuid), TYPE.STRING, 1.0)); + doc.addToFields(new Field("value", ByteBuffer.wrap("test".getBytes()), TYPE.STRING, 1.0)); + updatePackage.setTerm(new org.apache.blur.thrift.generated.Term("id", ByteBuffer.wrap(uuid))); + updatePackage.addToDocuments(doc); + updatePackages.add(updatePackage); + } + MutateOptions options = new MutateOptions(); + options.setTable(tableDescriptor.getName()); + options.setWaitToBeVisible(true); + options.setShardIndex(s); + client.updateDocuments(options, updatePackages); + } + + Session session = client.openReadSession(tableDescriptor.getName()); + QueryArgs queryArgs = new QueryArgs(); + Term term = new Term("value", "test"); + TermQuery query = new TermQuery(term); + queryArgs.setQuery(Convert.toBytes(query)); + List<TopFieldDocs> results = client.search(session, queryArgs); + long totalHits = 0; + for (TopFieldDocs fieldDocs : results) { + totalHits += fieldDocs.getTotalHits(); + } + assertEquals(SHARD_COUNT * 100, totalHits); + } +}
