http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java deleted file mode 100644 index 79e4b9f..0000000 --- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTest.java +++ /dev/null @@ -1,926 +0,0 @@ -package org.apache.blur.thrift; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.File; -import java.io.IOException; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryMXBean; -import java.lang.management.MemoryUsage; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.blur.MiniCluster; -import org.apache.blur.TestType; -import org.apache.blur.analysis.FieldManager; -import org.apache.blur.command.RunSlowForTesting; -import org.apache.blur.server.TableContext; -import org.apache.blur.thirdparty.thrift_0_9_0.TException; -import org.apache.blur.thrift.generated.Blur; -import org.apache.blur.thrift.generated.Blur.Iface; -import org.apache.blur.thrift.generated.BlurException; -import org.apache.blur.thrift.generated.BlurQuery; -import org.apache.blur.thrift.generated.BlurQueryStatus; -import org.apache.blur.thrift.generated.BlurResult; -import org.apache.blur.thrift.generated.BlurResults; -import org.apache.blur.thrift.generated.Column; -import org.apache.blur.thrift.generated.ColumnDefinition; -import org.apache.blur.thrift.generated.ErrorType; -import org.apache.blur.thrift.generated.Facet; -import org.apache.blur.thrift.generated.FetchResult; -import org.apache.blur.thrift.generated.Query; -import org.apache.blur.thrift.generated.QueryState; -import org.apache.blur.thrift.generated.Record; -import org.apache.blur.thrift.generated.RecordMutation; -import org.apache.blur.thrift.generated.RowMutation; -import org.apache.blur.thrift.generated.Schema; -import org.apache.blur.thrift.generated.Selector; -import org.apache.blur.thrift.generated.ShardState; -import org.apache.blur.thrift.generated.SortField; -import org.apache.blur.thrift.generated.TableDescriptor; -import org.apache.blur.thrift.generated.TableStats; -import org.apache.blur.thrift.util.BlurThriftHelper; -import org.apache.blur.user.User; -import org.apache.blur.user.UserContext; -import org.apache.blur.utils.BlurConstants; -import org.apache.blur.utils.GCWatcher; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.zookeeper.KeeperException; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public class BlurClusterTest { - - private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "./target/tmp_BlurClusterTest")); - private static MiniCluster miniCluster; - private static boolean externalProcesses = true; - - private int numberOfDocs = 1000; - private String controllerConnectionStr; - - @BeforeClass - public static void startCluster() throws IOException { - GCWatcher.init(0.60); - LocalFileSystem localFS = FileSystem.getLocal(new Configuration()); - File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile(); - testDirectory.mkdirs(); - - Path directory = new Path(testDirectory.getPath()); - FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission(); - FsAction userAction = dirPermissions.getUserAction(); - FsAction groupAction = dirPermissions.getGroupAction(); - FsAction otherAction = dirPermissions.getOtherAction(); - - StringBuilder builder = new StringBuilder(); - builder.append(userAction.ordinal()); - builder.append(groupAction.ordinal()); - builder.append(otherAction.ordinal()); - String dirPermissionNum = builder.toString(); - System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum); - testDirectory.delete(); - miniCluster = new MiniCluster(); - miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true, externalProcesses); - } - - @AfterClass - public static void shutdownCluster() { - miniCluster.shutdownBlurCluster(); - } - - @After - public void tearDown() throws BlurException, TException { - Iface client = getClient(); - List<String> tableList = client.tableList(); - for (String table : tableList) { - client.disableTable(table); - client.removeTable(table, true); - } - } - - private Iface getClient() { - if (controllerConnectionStr == null) { - controllerConnectionStr = miniCluster.getControllerConnectionStr(); - } - return BlurClient.getClient(controllerConnectionStr); - } - - @Test - public void testEnqueue() throws BlurException, TException, InterruptedException, IOException { - String tableName = "testEnqueue"; - createTable(tableName); - Blur.Iface client = getClient(); - - long s = System.currentTimeMillis(); - int count = 10000; - for (int i = 0; i < count; i++) { - String rowId = UUID.randomUUID().toString(); - RecordMutation mutation = BlurThriftHelper.newRecordMutation("test", rowId, - BlurThriftHelper.newColumn("test", "value")); - RowMutation rowMutation = BlurThriftHelper.newRowMutation(tableName, rowId, mutation); - client.enqueueMutate(rowMutation); - } - long e = System.currentTimeMillis(); - double seconds = (e - s) / 1000.0; - double rate = count / seconds; - System.out.println("Load row in queue at " + rate + "/s"); - - for (int i = 0; i < 60; i++) { - TableStats stats = client.tableStats(tableName); - long rowCount = stats.getRowCount(); - if (rowCount == count) { - return; - } - Thread.sleep(1000); - } - fail("Test failed to load all rows."); - } - - @Test - public void testBlurQueryWithRowId() throws BlurException, TException, InterruptedException, IOException { - String tableName = "testBlurQueryWithRowId"; - createTable(tableName); - loadTable(tableName); - Blur.Iface client = getClient(); - BlurQuery blurQuery = new BlurQuery(); - Query query = new Query(); - query.setQuery("*"); - blurQuery.setQuery(query); - BlurResults results1 = client.query(tableName, blurQuery); - assertEquals(numberOfDocs, results1.getTotalResults()); - String id1 = results1.getResults().iterator().next().getFetchResult().getRowResult().getRow().getId(); - - blurQuery.setRowId(id1); - - query.setRowQuery(false); - BlurResults results2 = client.query(tableName, blurQuery); - assertEquals(1, results2.getTotalResults()); - String id2 = results2.getResults().iterator().next().getFetchResult().getRecordResult().getRowid(); - - assertEquals(id1, id2); - System.out.println("Finished!"); - } - - @Test - public void testAdminCalls() throws BlurException, TException, IOException, InterruptedException { - String tableName = "testAdminCalls"; - createTable(tableName); - loadTable(tableName); - Blur.Iface client = getClient(); - List<String> shardClusterList = client.shardClusterList(); - assertEquals(1, shardClusterList.size()); - assertEquals(BlurConstants.DEFAULT, shardClusterList.get(0)); - - Map<String, String> shardServerLayout = client.shardServerLayout(tableName); - assertEquals(5, shardServerLayout.size()); - - Map<String, Map<String, ShardState>> shardServerLayoutState = client.shardServerLayoutState(tableName); - assertEquals(5, shardServerLayoutState.size()); - - List<String> shardServerList = client.shardServerList(BlurConstants.DEFAULT); - assertEquals(3, shardServerList.size()); - } - - @Test - public void testForEmptySchema() throws BlurException, TException, IOException, InterruptedException { - String tableName = "testForEmptySchema"; - createTable(tableName); - Blur.Iface client = getClient(); - Schema schema = client.schema(tableName); - Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies(); - assertTrue(families.isEmpty()); - - TableContext tableContext = TableContext.create(client.describe(tableName)); - FieldManager fieldManager = tableContext.getFieldManager(); - - assertTrue(fieldManager.addColumnDefinition("test-family", "test-column", null, false, "string", false, null)); - - TableContext.clear(); - Schema newschema = client.schema(tableName); - Map<String, Map<String, ColumnDefinition>> newfamilies = newschema.getFamilies(); - assertTrue(!newfamilies.isEmpty()); - int newsize = newfamilies.size(); - assertEquals(1, newsize); - } - - @Test - public void testCreateTableWithCustomType() throws IOException, BlurException, TException { - Blur.Iface client = getClient(); - TableDescriptor tableDescriptor = new TableDescriptor(); - tableDescriptor.setName("test_type"); - tableDescriptor.setShardCount(1); - tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/test_type"); - tableDescriptor.putToTableProperties("blur.fieldtype.customtype1", TestType.class.getName()); - client.createTable(tableDescriptor); - List<String> tableList = client.tableList(); - assertTrue(tableList.contains("test_type")); - - client.disableTable("test_type"); - - client.enableTable("test_type"); - - TableDescriptor describe = client.describe("test_type"); - Map<String, String> tableProperties = describe.getTableProperties(); - assertEquals(TestType.class.getName(), tableProperties.get("blur.fieldtype.customtype1")); - } - - public void createTable(String tableName) throws BlurException, TException, IOException { - Blur.Iface client = getClient(); - TableDescriptor tableDescriptor = new TableDescriptor(); - tableDescriptor.setName(tableName); - tableDescriptor.setShardCount(5); - tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/" + tableName); - client.createTable(tableDescriptor); - List<String> tableList = client.tableList(); - assertTrue(tableList.contains(tableName)); - } - - public void loadTable(String tableName) throws BlurException, TException, InterruptedException { - loadTable(tableName, 1); - } - - public void loadTable(String tableName, int pass) throws BlurException, TException, InterruptedException { - Iface client = getClient(); - int maxFacetValue = 100; - List<RowMutation> mutations = new ArrayList<RowMutation>(); - Random random = new Random(1); - for (int i = 0; i < numberOfDocs; i++) { - String rowId = UUID.randomUUID().toString(); - RecordMutation mutation = BlurThriftHelper.newRecordMutation("test", rowId, - BlurThriftHelper.newColumn("test", "value"), - BlurThriftHelper.newColumn("facet", Integer.toString(random.nextInt(maxFacetValue))), - BlurThriftHelper.newColumn("facetFixed", "test")); - RowMutation rowMutation = BlurThriftHelper.newRowMutation(tableName, rowId, mutation); - mutations.add(rowMutation); - } - ColumnDefinition columnDefinition = new ColumnDefinition(); - columnDefinition.setFamily("test"); - columnDefinition.setColumnName("facet"); - columnDefinition.setFieldLessIndexed(true); - columnDefinition.setFieldType("string"); - columnDefinition.setSortable(true); - columnDefinition.setProperties(new HashMap<String, String>()); - client.addColumnDefinition(tableName, columnDefinition); - long s = System.nanoTime(); - client.mutateBatch(mutations); - long e = System.nanoTime(); - System.out.println("mutateBatch took [" + (e - s) / 1000000.0 + "]"); - BlurQuery blurQueryRow = new BlurQuery(); - Query queryRow = new Query(); - queryRow.setQuery("test.test:value"); - blurQueryRow.setQuery(queryRow); - blurQueryRow.setUseCacheIfPresent(false); - blurQueryRow.setCacheResult(false); - BlurResults resultsRow = client.query(tableName, blurQueryRow); - assertRowResults(resultsRow); - assertEquals(numberOfDocs * pass, resultsRow.getTotalResults()); - - BlurQuery blurQueryRecord = new BlurQuery(); - Query queryRecord = new Query(); - queryRecord.rowQuery = false; - queryRecord.setQuery("test.test:value"); - blurQueryRecord.setQuery(queryRecord); - BlurResults resultsRecord = client.query(tableName, blurQueryRecord); - assertRecordResults(resultsRecord); - assertEquals(numberOfDocs * pass, resultsRecord.getTotalResults()); - - Schema schema = client.schema(tableName); - assertFalse(schema.getFamilies().isEmpty()); - } - - @Test - public void testQueryWithSelector() throws BlurException, TException, IOException, InterruptedException { - final String tableName = "testQueryWithSelector"; - createTable(tableName); - loadTable(tableName); - Iface client = getClient(); - BlurQuery blurQueryRow = new BlurQuery(); - Query queryRow = new Query(); - queryRow.setQuery("test.test:value"); - blurQueryRow.setQuery(queryRow); - blurQueryRow.setUseCacheIfPresent(false); - blurQueryRow.setCacheResult(false); - blurQueryRow.setSelector(new Selector()); - - BlurResults resultsRow = client.query(tableName, blurQueryRow); - // assertRowResults(resultsRow); - assertEquals(numberOfDocs, resultsRow.getTotalResults()); - - for (BlurResult blurResult : resultsRow.getResults()) { - System.out.println(blurResult); - } - - } - - @Test - public void testSortedQueryWithSelector() throws BlurException, TException, IOException, InterruptedException { - final String tableName = "testSortedQueryWithSelector"; - createTable(tableName); - loadTable(tableName); - - Iface client = getClient(); - - BlurQuery blurQueryRow = new BlurQuery(); - Query queryRow = new Query(); - queryRow.setQuery("test.test:value"); - queryRow.setRowQuery(false); - blurQueryRow.setQuery(queryRow); - blurQueryRow.addToSortFields(new SortField("test", "facet", false)); - - blurQueryRow.setUseCacheIfPresent(false); - blurQueryRow.setCacheResult(false); - Selector selector = new Selector(); - selector.setRecordOnly(true); - blurQueryRow.setSelector(selector); - - BlurResults resultsRow = client.query(tableName, blurQueryRow); - long totalResults = resultsRow.getTotalResults(); - - assertEquals(numberOfDocs, resultsRow.getTotalResults()); - - String lastValue = null; - long totalFetched = 0; - do { - for (BlurResult blurResult : resultsRow.getResults()) { - FetchResult fetchResult = blurResult.getFetchResult(); - Record record = fetchResult.getRecordResult().getRecord(); - if (lastValue == null) { - lastValue = getColumnValue(record, "facet"); - } else { - String currentValue = getColumnValue(record, "facet"); - if (currentValue.compareTo(lastValue) < 0) { - fail("Current Value of [" + currentValue + "] can not be less than lastValue of [" + lastValue + "]"); - } - lastValue = currentValue; - } - totalFetched++; - } - int size = resultsRow.getResults().size(); - totalResults -= size; - if (totalResults > 0) { - blurQueryRow.setStart(blurQueryRow.getStart() + size); - resultsRow = client.query(tableName, blurQueryRow); - } - } while (totalResults > 0); - assertEquals(numberOfDocs, totalFetched); - } - - private String getColumnValue(Record record, String columnName) { - for (Column col : record.getColumns()) { - if (col.getName().equals(columnName)) { - return col.getValue(); - } - } - return null; - } - - // @Test - public void testQueryWithSelectorForDeepPagingPerformance() throws BlurException, TException, IOException, - InterruptedException { - final String tableName = "testQueryWithSelectorForDeepPagingPerformance"; - createTable(tableName); - int passes = 10; - for (int i = 1; i <= passes; i++) { - loadTable(tableName, i); - } - Iface client = getClient(); - BlurQuery blurQueryRow = new BlurQuery(); - Query queryRow = new Query(); - queryRow.setQuery("test.test:value"); - blurQueryRow.setQuery(queryRow); - blurQueryRow.setUseCacheIfPresent(false); - blurQueryRow.setCacheResult(false); - blurQueryRow.setSelector(new Selector()); - - long start = System.nanoTime(); - int position = 0; - do { - blurQueryRow.setStart(position); - long s = System.nanoTime(); - BlurResults resultsRow = client.query(tableName, blurQueryRow); - long e = System.nanoTime(); - System.out.println("RUNNING QUERY.... starting at [" + position + "] took [" + (e - s) / 1000000.0 + " ms]"); - // assertRowResults(resultsRow); - assertEquals(numberOfDocs * passes, resultsRow.getTotalResults()); - - for (BlurResult blurResult : resultsRow.getResults()) { - System.out.println(blurResult); - position++; - } - } while (position < numberOfDocs * passes); - long end = System.nanoTime(); - System.out.println((end - start) / 1000000.0); - } - - @Test - public void testQueryWithFacets() throws BlurException, TException, IOException, InterruptedException { - final String tableName = "testQueryWithFacets"; - createTable(tableName); - loadTable(tableName); - Iface client = getClient(); - BlurQuery blurQueryRow = new BlurQuery(); - Query queryRow = new Query(); - // queryRow.setQuery("test.test:value"); - queryRow.setQuery("*"); - blurQueryRow.setQuery(queryRow); - blurQueryRow.setUseCacheIfPresent(false); - blurQueryRow.setCacheResult(false); - blurQueryRow.setSelector(new Selector()); - for (int i = 0; i < 250; i++) { - blurQueryRow.addToFacets(new Facet("test.facet:" + i, Long.MAX_VALUE)); - } - - BlurResults resultsRow = client.query(tableName, blurQueryRow); - assertEquals(numberOfDocs, resultsRow.getTotalResults()); - System.out.println(resultsRow.getFacetCounts()); - - System.out.println(); - - } - - @Test - public void testBatchFetch() throws BlurException, TException, InterruptedException, IOException { - String tableName = "testBatchFetch"; - createTable(tableName); - loadTable(tableName); - final Iface client = getClient(); - List<String> terms = client.terms(tableName, null, "rowid", "", (short) 100); - - List<Selector> selectors = new ArrayList<Selector>(); - for (String s : terms) { - Selector selector = new Selector(); - selector.setRowId(s); - selectors.add(selector); - } - - List<FetchResult> fetchRowBatch = client.fetchRowBatch(tableName, selectors); - assertEquals(100, fetchRowBatch.size()); - - int i = 0; - for (FetchResult fetchResult : fetchRowBatch) { - assertEquals(terms.get(i), fetchResult.getRowResult().getRow().getId()); - i++; - } - - } - - @Test - public void testQueryStatus() throws BlurException, TException, InterruptedException, IOException { - final String tableName = "testQueryStatus"; - createTable(tableName); - loadTable(tableName); - final Iface client = getClient(); - final BlurQuery blurQueryRow = new BlurQuery(); - Query queryRow = new Query(); - queryRow.setQuery("test.test:value"); - blurQueryRow.setQuery(queryRow); - blurQueryRow.setUseCacheIfPresent(false); - blurQueryRow.setCacheResult(false); - String uuid = "5678"; - blurQueryRow.setUuid(uuid); - final User user = new User("testuser", new HashMap<String, String>()); - - try { - setDebugRunSlow(tableName, true); - new Thread(new Runnable() { - @Override - public void run() { - try { - UserContext.setUser(user); - // This call will take several seconds to execute. - client.query(tableName, blurQueryRow); - } catch (BlurException e) { - // e.printStackTrace(); - } catch (TException e) { - e.printStackTrace(); - } - } - }).start(); - Thread.sleep(500); - BlurQueryStatus queryStatusById = client.queryStatusById(tableName, uuid); - assertEquals(user.getUsername(), queryStatusById.getUser().getUsername()); - assertEquals(queryStatusById.getState(), QueryState.RUNNING); - client.cancelQuery(tableName, uuid); - } finally { - setDebugRunSlow(tableName, false); - } - } - - @Test - public void testQueryCancel() throws BlurException, TException, InterruptedException, IOException { - final String tableName = "testQueryCancel"; - createTable(tableName); - loadTable(tableName); - final Iface client = getClient(); - try { - // This will make each collect in the collectors pause 250 ms per collect - // call - setDebugRunSlow(tableName, true); - final BlurQuery blurQueryRow = new BlurQuery(); - Query queryRow = new Query(); - queryRow.setQuery("test.test:value"); - blurQueryRow.setQuery(queryRow); - blurQueryRow.setUseCacheIfPresent(false); - blurQueryRow.setCacheResult(false); - blurQueryRow.setUuid("1234"); - - final AtomicReference<BlurException> error = new AtomicReference<BlurException>(); - final AtomicBoolean fail = new AtomicBoolean(); - - new Thread(new Runnable() { - @Override - public void run() { - try { - // This call will take several seconds to execute. - client.query(tableName, blurQueryRow); - fail.set(true); - } catch (BlurException e) { - error.set(e); - } catch (TException e) { - e.printStackTrace(); - fail.set(true); - } - } - }).start(); - Thread.sleep(500); - client.cancelQuery(tableName, blurQueryRow.getUuid()); - BlurException blurException = pollForError(error, 10, TimeUnit.SECONDS, null, fail, -1); - if (fail.get()) { - fail("Unknown error, failing test."); - } - assertEquals(blurException.getErrorType(), ErrorType.QUERY_CANCEL); - } finally { - setDebugRunSlow(tableName, false); - } - // Tests that the exitable reader was reset. - client.terms(tableName, "test", "facet", null, (short) 100); - } - - // @Test - public void testBackPressureViaQuery() throws BlurException, TException, InterruptedException, IOException { - // This will make each collect in the collectors pause 250 ms per collect - // call - String tableName = "testAdminCalls"; - createTable(tableName); - loadTable(tableName); - try { - setDebugRunSlow(tableName, true); - runBackPressureViaQuery(tableName); - Thread.sleep(1000); - System.gc(); - System.gc(); - Thread.sleep(1000); - } finally { - setDebugRunSlow(tableName, false); - } - } - - private void setDebugRunSlow(String table, boolean flag) throws IOException { - RunSlowForTesting runSlowForTesting = new RunSlowForTesting(); - runSlowForTesting.setRunSlow(flag); - runSlowForTesting.setTable(table); - runSlowForTesting.run(getClient()); - } - - private void runBackPressureViaQuery(final String tableName) throws InterruptedException { - final Iface client = getClient(); - final BlurQuery blurQueryRow = new BlurQuery(); - Query queryRow = new Query(); - queryRow.setQuery("test.test:value"); - blurQueryRow.setQuery(queryRow); - blurQueryRow.setUseCacheIfPresent(false); - blurQueryRow.setCacheResult(false); - blurQueryRow.setUuid("1234"); - - final AtomicReference<BlurException> error = new AtomicReference<BlurException>(); - final AtomicBoolean fail = new AtomicBoolean(); - - System.gc(); - System.gc(); - MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); - MemoryUsage usage = memoryMXBean.getHeapMemoryUsage(); - long max = usage.getMax(); - System.out.println("Max Heap [" + max + "]"); - long used = usage.getUsed(); - System.out.println("Used Heap [" + used + "]"); - long limit = (long) (max * 0.80); - System.out.println("Limit Heap [" + limit + "]"); - long difference = limit - used; - int sizeToAllocate = (int) ((int) difference * 0.50); - System.out.println("Allocating [" + sizeToAllocate + "] Heap [" + getHeapSize() + "] Max [" + getMaxHeapSize() - + "]"); - - byte[] bufferToFillHeap = new byte[sizeToAllocate]; - new Thread(new Runnable() { - @Override - public void run() { - try { - // This call will take several seconds to execute. - client.query(tableName, blurQueryRow); - fail.set(true); - } catch (BlurException e) { - System.out.println("-------------------"); - System.out.println("-------------------"); - System.out.println("-------------------"); - e.printStackTrace(); - System.out.println("-------------------"); - System.out.println("-------------------"); - System.out.println("-------------------"); - error.set(e); - } catch (TException e) { - e.printStackTrace(); - fail.set(true); - } - } - }).start(); - Thread.sleep(500); - List<byte[]> bufferToPutGcWatcherOverLimitList = new ArrayList<byte[]>(); - BlurException blurException = pollForError(error, 120, TimeUnit.SECONDS, bufferToPutGcWatcherOverLimitList, fail, - (int) (difference / 7)); - if (fail.get()) { - fail("Unknown error, failing test."); - } - System.out.println(bufferToFillHeap.hashCode()); - System.out.println(bufferToPutGcWatcherOverLimitList.hashCode()); - assertEquals(blurException.getErrorType(), ErrorType.BACK_PRESSURE); - bufferToPutGcWatcherOverLimitList.clear(); - bufferToPutGcWatcherOverLimitList = null; - bufferToFillHeap = null; - } - - private BlurException pollForError(AtomicReference<BlurException> error, long period, TimeUnit timeUnit, - List<byte[]> bufferToPutGcWatcherOverLimitList, AtomicBoolean fail, int sizeToAllocate) - throws InterruptedException { - long s = System.nanoTime(); - long totalTime = timeUnit.toNanos(period) + s; - if (bufferToPutGcWatcherOverLimitList != null) { - System.out.println("Allocating [" + sizeToAllocate + "] Heap [" + getHeapSize() + "] Max [" + getMaxHeapSize() - + "]"); - bufferToPutGcWatcherOverLimitList.add(new byte[sizeToAllocate]); - } - while (totalTime > System.nanoTime()) { - if (fail.get()) { - fail("The query failed."); - } - BlurException blurException = error.get(); - if (blurException != null) { - return blurException; - } - Thread.sleep(100); - if (bufferToPutGcWatcherOverLimitList != null) { - if (getHeapSize() < (getMaxHeapSize() * 0.8)) { - System.out.println("Allocating [" + sizeToAllocate + "] Heap [" + getHeapSize() + "] Max [" - + getMaxHeapSize() + "]"); - bufferToPutGcWatcherOverLimitList.add(new byte[sizeToAllocate]); - } else { - System.gc(); - System.gc(); - System.out.println("Already allocated enough Heap [" + getHeapSize() + "] Max [" + getMaxHeapSize() + "]"); - } - } - } - return null; - } - - private long getHeapSize() { - return ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed(); - } - - private long getMaxHeapSize() { - return ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); - } - - @Test - public void testTestShardFailover() throws BlurException, TException, InterruptedException, IOException, - KeeperException { - final String tableName = "testTestShardFailover"; - createTable(tableName); - loadTable(tableName); - Iface client = getClient(); - BlurQuery blurQuery = new BlurQuery(); - blurQuery.setUseCacheIfPresent(false); - Query query = new Query(); - query.setQuery("test.test:value"); - blurQuery.setQuery(query); - BlurResults results1 = client.query(tableName, blurQuery); - assertEquals(numberOfDocs, results1.getTotalResults()); - assertRowResults(results1); - - miniCluster.killShardServer(1); - - // make sure the WAL syncs - Thread.sleep(TimeUnit.SECONDS.toMillis(1)); - - // This should block until shards have failed over - client.shardServerLayout(tableName); - - assertEquals("We should have lost a node.", 2, client.shardServerList(BlurConstants.DEFAULT).size()); - assertEquals(numberOfDocs, client.query(tableName, blurQuery).getTotalResults()); - - miniCluster.startShards(1, true, externalProcesses); - Thread.sleep(TimeUnit.SECONDS.toMillis(1)); - - assertEquals("We should have the cluster back where we started.", 3, client.shardServerList(BlurConstants.DEFAULT) - .size()); - } - - @Test - public void testTrucateRaceCondition() throws BlurException, TException, IOException, InterruptedException { - String tableName = "testTrucateRaceCondition"; - createTable(tableName); - loadTable(tableName); - List<Connection> connections = BlurClientManager.getConnections(miniCluster.getControllerConnectionStr()); - Iface client1 = BlurClient.getClient(connections.get(0)); - Iface client2 = BlurClient.getClient(connections.get(1)); - TableDescriptor describe = client1.describe(tableName); - client1.disableTable(tableName); - client1.removeTable(tableName, true); - client1.createTable(describe); - - String rowId = UUID.randomUUID().toString(); - RecordMutation mutation = BlurThriftHelper.newRecordMutation("test", rowId, - BlurThriftHelper.newColumn("test", "value"), BlurThriftHelper.newColumn("facetFixed", "test")); - RowMutation rowMutation = BlurThriftHelper.newRowMutation(tableName, rowId, mutation); - client2.mutate(rowMutation); - } - - @Test - public void testTableRemovalWithFieldDefsV1() throws BlurException, TException, IOException, InterruptedException { - String tableName = "testTableRemovalWithFieldDefsV1"; - createTable(tableName); - loadTable(tableName); - - String family = "testTableRemovalWithFieldDefs-fam"; - - List<Connection> connections = BlurClientManager.getConnections(miniCluster.getControllerConnectionStr()); - Iface client1 = BlurClient.getClient(connections.get(0)); - ColumnDefinition columnDefinition = new ColumnDefinition(); - columnDefinition.setColumnName("col"); - columnDefinition.setFamily(family); - columnDefinition.setFieldLessIndexed(false); - columnDefinition.setFieldType("string"); - columnDefinition.setSortable(false); - columnDefinition.setSubColumnName(null); - assertTrue(client1.addColumnDefinition(tableName, columnDefinition)); - - client1.disableTable(tableName); - client1.removeTable(tableName, true); - - createTable(tableName); - - Iface client2 = BlurClient.getClient(connections.get(1)); - assertFamilyIsNotPresent(tableName, client2, family); - - List<String> shardClusterList = client2.shardClusterList(); - - for (String cluster : shardClusterList) { - List<String> shardServerList = client2.shardServerList(cluster); - for (String shardServer : shardServerList) { - Iface client = BlurClient.getClient(shardServer); - assertFamilyIsNotPresent(tableName, client, family); - } - } - - } - - @Test - public void testTableRemovalWithFieldDefsV2() throws BlurException, TException, IOException, InterruptedException { - String tableName = "testTableRemovalWithFieldDefsV2"; - createTable(tableName); - loadTable(tableName); - - Iface client = getClient(); - Schema expectedSchema = client.schema(tableName); - Set<String> expectedFiles = new TreeSet<String>(); - TableDescriptor describe = client.describe(tableName); - - { - Path path = new Path(describe.getTableUri()); - FileSystem fileSystem = path.getFileSystem(new Configuration()); - FileStatus[] listStatus = fileSystem.listStatus(new Path(path, "types")); - for (FileStatus fileStatus : listStatus) { - System.out.println("Expected " + fileStatus.getPath()); - expectedFiles.add(fileStatus.getPath().toString()); - } - } - - client.disableTable(tableName); - client.removeTable(tableName, true); - - createTable(tableName); - loadTable(tableName); - Schema actualSchema = client.schema(tableName); - - assertEquals(expectedSchema.getTable(), actualSchema.getTable()); - Map<String, Map<String, ColumnDefinition>> expectedFamilies = expectedSchema.getFamilies(); - Map<String, Map<String, ColumnDefinition>> actualFamilies = actualSchema.getFamilies(); - assertEquals(new TreeSet<String>(expectedFamilies.keySet()), new TreeSet<String>(actualFamilies.keySet())); - - for (String family : expectedFamilies.keySet()) { - Map<String, ColumnDefinition> expectedColDefMap = new TreeMap<String, ColumnDefinition>( - expectedFamilies.get(family)); - Map<String, ColumnDefinition> actualColDefMap = new TreeMap<String, ColumnDefinition>(actualFamilies.get(family)); - assertEquals(expectedColDefMap, actualColDefMap); - } - - System.out.println(expectedSchema); - - Set<String> actualFiles = new TreeSet<String>(); - { - Path path = new Path(describe.getTableUri()); - FileSystem fileSystem = path.getFileSystem(new Configuration()); - FileStatus[] listStatus = fileSystem.listStatus(new Path(path, "types")); - for (FileStatus fileStatus : listStatus) { - System.out.println("Actual " + fileStatus.getPath()); - actualFiles.add(fileStatus.getPath().toString()); - } - } - assertEquals(expectedFiles, actualFiles); - } - - private void assertFamilyIsNotPresent(String tableName, Iface client, String family) throws BlurException, TException { - Schema schema = client.schema(tableName); - Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies(); - assertNull(families.get(family)); - } - - private void assertRowResults(BlurResults results) { - for (BlurResult result : results.getResults()) { - assertNull(result.locationId); - assertNull(result.fetchResult.recordResult); - assertNull(result.fetchResult.rowResult.row.records); - assertNotNull(result.fetchResult.rowResult.row.id); - } - } - - private void assertRecordResults(BlurResults results) { - for (BlurResult result : results.getResults()) { - assertNull(result.locationId); - assertNotNull(result.fetchResult.recordResult); - assertNotNull(result.fetchResult.recordResult.rowid); - assertNotNull(result.fetchResult.recordResult.record.recordId); - assertNotNull(result.fetchResult.recordResult.record.family); - assertNull("Not null [" + result.fetchResult.recordResult.record.columns + "]", - result.fetchResult.recordResult.record.columns); - assertNull(result.fetchResult.rowResult); - } - } - - @Test - public void testCreateDisableAndRemoveTable() throws IOException, BlurException, TException { - Iface client = getClient(); - String tableName = UUID.randomUUID().toString(); - TableDescriptor tableDescriptor = new TableDescriptor(); - tableDescriptor.setName(tableName); - tableDescriptor.setShardCount(5); - tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/" + tableName); - - for (int i = 0; i < 3; i++) { - client.createTable(tableDescriptor); - client.disableTable(tableName); - client.removeTable(tableName, true); - } - - assertFalse(client.tableList().contains(tableName)); - - } -}
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java new file mode 100644 index 0000000..83d5adc --- /dev/null +++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java @@ -0,0 +1,965 @@ +package org.apache.blur.thrift; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.blur.MiniCluster; +import org.apache.blur.TestType; +import org.apache.blur.analysis.FieldManager; +import org.apache.blur.command.RunSlowForTesting; +import org.apache.blur.server.TableContext; +import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thrift.generated.Blur; +import org.apache.blur.thrift.generated.Blur.Iface; +import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.thrift.generated.BlurQuery; +import org.apache.blur.thrift.generated.BlurQueryStatus; +import org.apache.blur.thrift.generated.BlurResult; +import org.apache.blur.thrift.generated.BlurResults; +import org.apache.blur.thrift.generated.Column; +import org.apache.blur.thrift.generated.ColumnDefinition; +import org.apache.blur.thrift.generated.ErrorType; +import org.apache.blur.thrift.generated.Facet; +import org.apache.blur.thrift.generated.FetchResult; +import org.apache.blur.thrift.generated.Query; +import org.apache.blur.thrift.generated.QueryState; +import org.apache.blur.thrift.generated.Record; +import org.apache.blur.thrift.generated.RecordMutation; +import org.apache.blur.thrift.generated.RowMutation; +import org.apache.blur.thrift.generated.Schema; +import org.apache.blur.thrift.generated.Selector; +import org.apache.blur.thrift.generated.ShardState; +import org.apache.blur.thrift.generated.SortField; +import org.apache.blur.thrift.generated.TableDescriptor; +import org.apache.blur.thrift.generated.TableStats; +import org.apache.blur.thrift.util.BlurThriftHelper; +import org.apache.blur.user.User; +import org.apache.blur.user.UserContext; +import org.apache.blur.utils.BlurConstants; +import org.apache.blur.utils.GCWatcher; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.zookeeper.KeeperException; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public abstract class BlurClusterTestBase { + + private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "./target/tmp_BlurClusterTest")); + private static MiniCluster miniCluster; + private static boolean externalProcesses = false; + + private int numberOfDocs = 1000; + private String controllerConnectionStr; + + @BeforeClass + public static void startCluster() throws IOException { + GCWatcher.init(0.60); + LocalFileSystem localFS = FileSystem.getLocal(new Configuration()); + File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile(); + testDirectory.mkdirs(); + + Path directory = new Path(testDirectory.getPath()); + FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission(); + FsAction userAction = dirPermissions.getUserAction(); + FsAction groupAction = dirPermissions.getGroupAction(); + FsAction otherAction = dirPermissions.getOtherAction(); + + StringBuilder builder = new StringBuilder(); + builder.append(userAction.ordinal()); + builder.append(groupAction.ordinal()); + builder.append(otherAction.ordinal()); + String dirPermissionNum = builder.toString(); + System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum); + testDirectory.delete(); + miniCluster = new MiniCluster(); + miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true, externalProcesses); + } + + @AfterClass + public static void shutdownCluster() { + miniCluster.shutdownBlurCluster(); + } + + @Before + public void setup() { + UserContext.setUser(getUser()); + } + + protected abstract User getUser(); + + @After + public void tearDown() throws BlurException, TException { + Iface client = getClient(); + List<String> tableList = client.tableList(); + for (String table : tableList) { + client.disableTable(table); + client.removeTable(table, true); + } + } + + private Iface getClient() { + if (controllerConnectionStr == null) { + controllerConnectionStr = miniCluster.getControllerConnectionStr(); + } + return BlurClient.getClient(controllerConnectionStr); + } + + @Test + public void testEnqueue() throws BlurException, TException, InterruptedException, IOException { + String tableName = "testEnqueue"; + createTable(tableName); + Blur.Iface client = getClient(); + + long s = System.currentTimeMillis(); + int count = 10000; + for (int i = 0; i < count; i++) { + String rowId = UUID.randomUUID().toString(); + RecordMutation mutation = BlurThriftHelper.newRecordMutation("test", rowId, + BlurThriftHelper.newColumn("test", "value")); + RowMutation rowMutation = BlurThriftHelper.newRowMutation(tableName, rowId, mutation); + client.enqueueMutate(mutate(rowMutation)); + } + long e = System.currentTimeMillis(); + double seconds = (e - s) / 1000.0; + double rate = count / seconds; + System.out.println("Load row in queue at " + rate + "/s"); + + for (int i = 0; i < 60; i++) { + TableStats stats = client.tableStats(tableName); + long rowCount = stats.getRowCount(); + if (rowCount == count) { + return; + } + Thread.sleep(1000); + } + fail("Test failed to load all rows."); + } + + protected abstract RowMutation mutate(RowMutation rowMutation); + + @Test + public void testBlurQueryWithRowId() throws BlurException, TException, InterruptedException, IOException { + String tableName = "testBlurQueryWithRowId"; + createTable(tableName); + loadTable(tableName); + Blur.Iface client = getClient(); + BlurQuery blurQuery = new BlurQuery(); + Query query = new Query(); + query.setQuery("*"); + blurQuery.setQuery(query); + BlurResults results1 = client.query(tableName, blurQuery); + assertEquals(numberOfDocs, results1.getTotalResults()); + String id1 = results1.getResults().iterator().next().getFetchResult().getRowResult().getRow().getId(); + + blurQuery.setRowId(id1); + + query.setRowQuery(false); + BlurResults results2 = client.query(tableName, blurQuery); + assertEquals(1, results2.getTotalResults()); + String id2 = results2.getResults().iterator().next().getFetchResult().getRecordResult().getRowid(); + + assertEquals(id1, id2); + System.out.println("Finished!"); + } + + @Test + public void testAdminCalls() throws BlurException, TException, IOException, InterruptedException { + String tableName = "testAdminCalls"; + createTable(tableName); + loadTable(tableName); + Blur.Iface client = getClient(); + List<String> shardClusterList = client.shardClusterList(); + assertEquals(1, shardClusterList.size()); + assertEquals(BlurConstants.DEFAULT, shardClusterList.get(0)); + + Map<String, String> shardServerLayout = client.shardServerLayout(tableName); + assertEquals(5, shardServerLayout.size()); + + Map<String, Map<String, ShardState>> shardServerLayoutState = client.shardServerLayoutState(tableName); + assertEquals(5, shardServerLayoutState.size()); + + List<String> shardServerList = client.shardServerList(BlurConstants.DEFAULT); + assertEquals(3, shardServerList.size()); + } + + @Test + public void testForEmptySchema() throws BlurException, TException, IOException, InterruptedException { + String tableName = "testForEmptySchema"; + createTable(tableName, false); + Blur.Iface client = getClient(); + Schema schema = client.schema(tableName); + Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies(); + assertTrue(families.isEmpty()); + + TableContext tableContext = TableContext.create(client.describe(tableName)); + FieldManager fieldManager = tableContext.getFieldManager(); + + assertTrue(fieldManager.addColumnDefinition("test-family", "test-column", null, false, "string", false, null)); + + TableContext.clear(); + Schema newschema = client.schema(tableName); + Map<String, Map<String, ColumnDefinition>> newfamilies = newschema.getFamilies(); + assertTrue(!newfamilies.isEmpty()); + int newsize = newfamilies.size(); + assertEquals(1, newsize); + } + + @Test + public void testCreateTableWithCustomType() throws IOException, BlurException, TException { + Blur.Iface client = getClient(); + TableDescriptor tableDescriptor = new TableDescriptor(); + tableDescriptor.setName("test_type"); + tableDescriptor.setShardCount(1); + tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/test_type"); + tableDescriptor.putToTableProperties("blur.fieldtype.customtype1", TestType.class.getName()); + setupTableProperties(tableDescriptor); + client.createTable(tableDescriptor); + postTableCreate(tableDescriptor, client); + List<String> tableList = client.tableList(); + assertTrue(tableList.contains("test_type")); + + client.disableTable("test_type"); + + client.enableTable("test_type"); + + TableDescriptor describe = client.describe("test_type"); + Map<String, String> tableProperties = describe.getTableProperties(); + assertEquals(TestType.class.getName(), tableProperties.get("blur.fieldtype.customtype1")); + } + + public void createTable(String tableName) throws BlurException, TException, IOException { + createTable(tableName, true); + } + + public void createTable(String tableName, boolean applyAcl) throws BlurException, TException, IOException { + Blur.Iface client = getClient(); + TableDescriptor tableDescriptor = new TableDescriptor(); + tableDescriptor.setName(tableName); + tableDescriptor.setShardCount(5); + tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/" + tableName); + setupTableProperties(tableDescriptor); + client.createTable(tableDescriptor); + if (applyAcl) { + postTableCreate(tableDescriptor, client); + } + List<String> tableList = client.tableList(); + assertTrue(tableList.contains(tableName)); + } + + protected abstract void postTableCreate(TableDescriptor tableDescriptor, Iface client); + + protected abstract void setupTableProperties(TableDescriptor tableDescriptor); + + public void loadTable(String tableName) throws BlurException, TException, InterruptedException { + loadTable(tableName, 1); + } + + public void loadTable(String tableName, int pass) throws BlurException, TException, InterruptedException { + Iface client = getClient(); + int maxFacetValue = 100; + List<RowMutation> mutations = new ArrayList<RowMutation>(); + Random random = new Random(1); + for (int i = 0; i < numberOfDocs; i++) { + String rowId = UUID.randomUUID().toString(); + RecordMutation mutation = BlurThriftHelper.newRecordMutation("test", rowId, + BlurThriftHelper.newColumn("test", "value"), + BlurThriftHelper.newColumn("facet", Integer.toString(random.nextInt(maxFacetValue))), + BlurThriftHelper.newColumn("facetFixed", "test")); + RowMutation rowMutation = BlurThriftHelper.newRowMutation(tableName, rowId, mutation); + mutations.add(rowMutation); + } + ColumnDefinition columnDefinition = new ColumnDefinition(); + columnDefinition.setFamily("test"); + columnDefinition.setColumnName("facet"); + columnDefinition.setFieldLessIndexed(true); + columnDefinition.setFieldType("string"); + columnDefinition.setSortable(true); + columnDefinition.setProperties(new HashMap<String, String>()); + client.addColumnDefinition(tableName, columnDefinition); + long s = System.nanoTime(); + client.mutateBatch(mutate(mutations)); + long e = System.nanoTime(); + System.out.println("mutateBatch took [" + (e - s) / 1000000.0 + "]"); + BlurQuery blurQueryRow = new BlurQuery(); + Query queryRow = new Query(); + queryRow.setQuery("test.test:value"); + blurQueryRow.setQuery(queryRow); + blurQueryRow.setUseCacheIfPresent(false); + blurQueryRow.setCacheResult(false); + BlurResults resultsRow = client.query(tableName, blurQueryRow); + assertRowResults(resultsRow); + assertEquals(numberOfDocs * pass, resultsRow.getTotalResults()); + + BlurQuery blurQueryRecord = new BlurQuery(); + Query queryRecord = new Query(); + queryRecord.rowQuery = false; + queryRecord.setQuery("test.test:value"); + blurQueryRecord.setQuery(queryRecord); + BlurResults resultsRecord = client.query(tableName, blurQueryRecord); + assertRecordResults(resultsRecord); + assertEquals(numberOfDocs * pass, resultsRecord.getTotalResults()); + + Schema schema = client.schema(tableName); + assertFalse(schema.getFamilies().isEmpty()); + } + + protected List<RowMutation> mutate(List<RowMutation> mutations) { + List<RowMutation> rowMutations = new ArrayList<RowMutation>(); + for (int i = 0; i < mutations.size(); i++) { + rowMutations.add(mutate(mutations.get(i))); + } + return rowMutations; + } + + @Test + public void testQueryWithSelector() throws BlurException, TException, IOException, InterruptedException { + final String tableName = "testQueryWithSelector"; + createTable(tableName); + loadTable(tableName); + Iface client = getClient(); + BlurQuery blurQueryRow = new BlurQuery(); + Query queryRow = new Query(); + queryRow.setQuery("test.test:value"); + blurQueryRow.setQuery(queryRow); + blurQueryRow.setUseCacheIfPresent(false); + blurQueryRow.setCacheResult(false); + blurQueryRow.setSelector(new Selector()); + + BlurResults resultsRow = client.query(tableName, blurQueryRow); + // assertRowResults(resultsRow); + assertEquals(numberOfDocs, resultsRow.getTotalResults()); + + for (BlurResult blurResult : resultsRow.getResults()) { + System.out.println(blurResult); + } + + } + + @Test + public void testSortedQueryWithSelector() throws BlurException, TException, IOException, InterruptedException { + final String tableName = "testSortedQueryWithSelector"; + createTable(tableName); + loadTable(tableName); + + Iface client = getClient(); + + BlurQuery blurQueryRow = new BlurQuery(); + Query queryRow = new Query(); + queryRow.setQuery("test.test:value"); + queryRow.setRowQuery(false); + blurQueryRow.setQuery(queryRow); + blurQueryRow.addToSortFields(new SortField("test", "facet", false)); + + blurQueryRow.setUseCacheIfPresent(false); + blurQueryRow.setCacheResult(false); + Selector selector = new Selector(); + selector.setRecordOnly(true); + blurQueryRow.setSelector(selector); + + BlurResults resultsRow = client.query(tableName, blurQueryRow); + long totalResults = resultsRow.getTotalResults(); + + assertEquals(numberOfDocs, resultsRow.getTotalResults()); + + String lastValue = null; + long totalFetched = 0; + do { + for (BlurResult blurResult : resultsRow.getResults()) { + FetchResult fetchResult = blurResult.getFetchResult(); + Record record = fetchResult.getRecordResult().getRecord(); + if (lastValue == null) { + lastValue = getColumnValue(record, "facet"); + } else { + String currentValue = getColumnValue(record, "facet"); + if (currentValue.compareTo(lastValue) < 0) { + fail("Current Value of [" + currentValue + "] can not be less than lastValue of [" + lastValue + "]"); + } + lastValue = currentValue; + } + totalFetched++; + } + int size = resultsRow.getResults().size(); + totalResults -= size; + if (totalResults > 0) { + blurQueryRow.setStart(blurQueryRow.getStart() + size); + resultsRow = client.query(tableName, blurQueryRow); + } + } while (totalResults > 0); + assertEquals(numberOfDocs, totalFetched); + } + + private String getColumnValue(Record record, String columnName) { + for (Column col : record.getColumns()) { + if (col.getName().equals(columnName)) { + return col.getValue(); + } + } + return null; + } + + // @Test + public void testQueryWithSelectorForDeepPagingPerformance() throws BlurException, TException, IOException, + InterruptedException { + final String tableName = "testQueryWithSelectorForDeepPagingPerformance"; + createTable(tableName); + int passes = 10; + for (int i = 1; i <= passes; i++) { + loadTable(tableName, i); + } + Iface client = getClient(); + BlurQuery blurQueryRow = new BlurQuery(); + Query queryRow = new Query(); + queryRow.setQuery("test.test:value"); + blurQueryRow.setQuery(queryRow); + blurQueryRow.setUseCacheIfPresent(false); + blurQueryRow.setCacheResult(false); + blurQueryRow.setSelector(new Selector()); + + long start = System.nanoTime(); + int position = 0; + do { + blurQueryRow.setStart(position); + long s = System.nanoTime(); + BlurResults resultsRow = client.query(tableName, blurQueryRow); + long e = System.nanoTime(); + System.out.println("RUNNING QUERY.... starting at [" + position + "] took [" + (e - s) / 1000000.0 + " ms]"); + // assertRowResults(resultsRow); + assertEquals(numberOfDocs * passes, resultsRow.getTotalResults()); + + for (BlurResult blurResult : resultsRow.getResults()) { + System.out.println(blurResult); + position++; + } + } while (position < numberOfDocs * passes); + long end = System.nanoTime(); + System.out.println((end - start) / 1000000.0); + } + + @Test + public void testQueryWithFacets() throws BlurException, TException, IOException, InterruptedException { + final String tableName = "testQueryWithFacets"; + createTable(tableName); + loadTable(tableName); + Iface client = getClient(); + BlurQuery blurQueryRow = new BlurQuery(); + Query queryRow = new Query(); + // queryRow.setQuery("test.test:value"); + queryRow.setQuery("*"); + blurQueryRow.setQuery(queryRow); + blurQueryRow.setUseCacheIfPresent(false); + blurQueryRow.setCacheResult(false); + blurQueryRow.setSelector(new Selector()); + for (int i = 0; i < 250; i++) { + blurQueryRow.addToFacets(new Facet("test.facet:" + i, Long.MAX_VALUE)); + } + + BlurResults resultsRow = client.query(tableName, blurQueryRow); + assertEquals(numberOfDocs, resultsRow.getTotalResults()); + System.out.println(resultsRow.getFacetCounts()); + + System.out.println(); + + } + + @Test + public void testBatchFetch() throws BlurException, TException, InterruptedException, IOException { + String tableName = "testBatchFetch"; + createTable(tableName); + loadTable(tableName); + final Iface client = getClient(); + List<String> terms = client.terms(tableName, null, "rowid", "", (short) 100); + + List<Selector> selectors = new ArrayList<Selector>(); + for (String s : terms) { + Selector selector = new Selector(); + selector.setRowId(s); + selectors.add(selector); + } + + List<FetchResult> fetchRowBatch = client.fetchRowBatch(tableName, selectors); + assertEquals(100, fetchRowBatch.size()); + + int i = 0; + for (FetchResult fetchResult : fetchRowBatch) { + assertEquals(terms.get(i), fetchResult.getRowResult().getRow().getId()); + i++; + } + + } + + @Test + public void testQueryStatus() throws BlurException, TException, InterruptedException, IOException { + final String tableName = "testQueryStatus"; + createTable(tableName); + loadTable(tableName); + final Iface client = getClient(); + final BlurQuery blurQueryRow = new BlurQuery(); + Query queryRow = new Query(); + queryRow.setQuery("test.test:value"); + blurQueryRow.setQuery(queryRow); + blurQueryRow.setUseCacheIfPresent(false); + blurQueryRow.setCacheResult(false); + String uuid = "5678"; + blurQueryRow.setUuid(uuid); + final User user = new User("testuser", getUserAttributes()); + try { + setDebugRunSlow(tableName, true); + new Thread(new Runnable() { + @Override + public void run() { + try { + UserContext.setUser(user); + // This call will take several seconds to execute. + client.query(tableName, blurQueryRow); + } catch (BlurException e) { + // e.printStackTrace(); + } catch (TException e) { + e.printStackTrace(); + } + } + }).start(); + Thread.sleep(500); + BlurQueryStatus queryStatusById = client.queryStatusById(tableName, uuid); + assertEquals(user.getUsername(), queryStatusById.getUser().getUsername()); + assertEquals(queryStatusById.getState(), QueryState.RUNNING); + client.cancelQuery(tableName, uuid); + } finally { + setDebugRunSlow(tableName, false); + } + } + + protected abstract Map<String, String> getUserAttributes(); + + @Test + public void testQueryCancel() throws BlurException, TException, InterruptedException, IOException { + final String tableName = "testQueryCancel"; + createTable(tableName); + loadTable(tableName); + final Iface client = getClient(); + try { + // This will make each collect in the collectors pause 250 ms per collect + // call + setDebugRunSlow(tableName, true); + final BlurQuery blurQueryRow = new BlurQuery(); + Query queryRow = new Query(); + queryRow.setQuery("test.test:value"); + blurQueryRow.setQuery(queryRow); + blurQueryRow.setUseCacheIfPresent(false); + blurQueryRow.setCacheResult(false); + blurQueryRow.setUuid("1234"); + + final AtomicReference<BlurException> error = new AtomicReference<BlurException>(); + final AtomicBoolean fail = new AtomicBoolean(); + final User user = UserContext.getUser(); + new Thread(new Runnable() { + @Override + public void run() { + try { + UserContext.setUser(user); + // This call will take several seconds to execute. + client.query(tableName, blurQueryRow); + fail.set(true); + } catch (BlurException e) { + error.set(e); + } catch (TException e) { + e.printStackTrace(); + fail.set(true); + } + } + }).start(); + Thread.sleep(500); + client.cancelQuery(tableName, blurQueryRow.getUuid()); + BlurException blurException = pollForError(error, 10, TimeUnit.SECONDS, null, fail, -1); + if (fail.get()) { + fail("Unknown error, failing test."); + } + assertEquals(blurException.getErrorType(), ErrorType.QUERY_CANCEL); + } finally { + setDebugRunSlow(tableName, false); + } + // Tests that the exitable reader was reset. + client.terms(tableName, "test", "facet", null, (short) 100); + } + + // @Test + public void testBackPressureViaQuery() throws BlurException, TException, InterruptedException, IOException { + // This will make each collect in the collectors pause 250 ms per collect + // call + String tableName = "testAdminCalls"; + createTable(tableName); + loadTable(tableName); + try { + setDebugRunSlow(tableName, true); + runBackPressureViaQuery(tableName); + Thread.sleep(1000); + System.gc(); + System.gc(); + Thread.sleep(1000); + } finally { + setDebugRunSlow(tableName, false); + } + } + + private void setDebugRunSlow(String table, boolean flag) throws IOException { + RunSlowForTesting runSlowForTesting = new RunSlowForTesting(); + runSlowForTesting.setRunSlow(flag); + runSlowForTesting.setTable(table); + runSlowForTesting.run(getClient()); + } + + private void runBackPressureViaQuery(final String tableName) throws InterruptedException { + final Iface client = getClient(); + final BlurQuery blurQueryRow = new BlurQuery(); + Query queryRow = new Query(); + queryRow.setQuery("test.test:value"); + blurQueryRow.setQuery(queryRow); + blurQueryRow.setUseCacheIfPresent(false); + blurQueryRow.setCacheResult(false); + blurQueryRow.setUuid("1234"); + + final AtomicReference<BlurException> error = new AtomicReference<BlurException>(); + final AtomicBoolean fail = new AtomicBoolean(); + + System.gc(); + System.gc(); + MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); + MemoryUsage usage = memoryMXBean.getHeapMemoryUsage(); + long max = usage.getMax(); + System.out.println("Max Heap [" + max + "]"); + long used = usage.getUsed(); + System.out.println("Used Heap [" + used + "]"); + long limit = (long) (max * 0.80); + System.out.println("Limit Heap [" + limit + "]"); + long difference = limit - used; + int sizeToAllocate = (int) ((int) difference * 0.50); + System.out.println("Allocating [" + sizeToAllocate + "] Heap [" + getHeapSize() + "] Max [" + getMaxHeapSize() + + "]"); + + byte[] bufferToFillHeap = new byte[sizeToAllocate]; + final User user = UserContext.getUser(); + new Thread(new Runnable() { + @Override + public void run() { + UserContext.setUser(user); + try { + // This call will take several seconds to execute. + client.query(tableName, blurQueryRow); + fail.set(true); + } catch (BlurException e) { + System.out.println("-------------------"); + System.out.println("-------------------"); + System.out.println("-------------------"); + e.printStackTrace(); + System.out.println("-------------------"); + System.out.println("-------------------"); + System.out.println("-------------------"); + error.set(e); + } catch (TException e) { + e.printStackTrace(); + fail.set(true); + } + } + }).start(); + Thread.sleep(500); + List<byte[]> bufferToPutGcWatcherOverLimitList = new ArrayList<byte[]>(); + BlurException blurException = pollForError(error, 120, TimeUnit.SECONDS, bufferToPutGcWatcherOverLimitList, fail, + (int) (difference / 7)); + if (fail.get()) { + fail("Unknown error, failing test."); + } + System.out.println(bufferToFillHeap.hashCode()); + System.out.println(bufferToPutGcWatcherOverLimitList.hashCode()); + assertEquals(blurException.getErrorType(), ErrorType.BACK_PRESSURE); + bufferToPutGcWatcherOverLimitList.clear(); + bufferToPutGcWatcherOverLimitList = null; + bufferToFillHeap = null; + } + + private BlurException pollForError(AtomicReference<BlurException> error, long period, TimeUnit timeUnit, + List<byte[]> bufferToPutGcWatcherOverLimitList, AtomicBoolean fail, int sizeToAllocate) + throws InterruptedException { + long s = System.nanoTime(); + long totalTime = timeUnit.toNanos(period) + s; + if (bufferToPutGcWatcherOverLimitList != null) { + System.out.println("Allocating [" + sizeToAllocate + "] Heap [" + getHeapSize() + "] Max [" + getMaxHeapSize() + + "]"); + bufferToPutGcWatcherOverLimitList.add(new byte[sizeToAllocate]); + } + while (totalTime > System.nanoTime()) { + if (fail.get()) { + fail("The query failed."); + } + BlurException blurException = error.get(); + if (blurException != null) { + return blurException; + } + Thread.sleep(100); + if (bufferToPutGcWatcherOverLimitList != null) { + if (getHeapSize() < (getMaxHeapSize() * 0.8)) { + System.out.println("Allocating [" + sizeToAllocate + "] Heap [" + getHeapSize() + "] Max [" + + getMaxHeapSize() + "]"); + bufferToPutGcWatcherOverLimitList.add(new byte[sizeToAllocate]); + } else { + System.gc(); + System.gc(); + System.out.println("Already allocated enough Heap [" + getHeapSize() + "] Max [" + getMaxHeapSize() + "]"); + } + } + } + return null; + } + + private long getHeapSize() { + return ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed(); + } + + private long getMaxHeapSize() { + return ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); + } + + @Test + public void testTestShardFailover() throws BlurException, TException, InterruptedException, IOException, + KeeperException { + final String tableName = "testTestShardFailover"; + createTable(tableName); + loadTable(tableName); + Iface client = getClient(); + BlurQuery blurQuery = new BlurQuery(); + blurQuery.setUseCacheIfPresent(false); + Query query = new Query(); + query.setQuery("test.test:value"); + blurQuery.setQuery(query); + BlurResults results1 = client.query(tableName, blurQuery); + assertEquals(numberOfDocs, results1.getTotalResults()); + assertRowResults(results1); + + miniCluster.killShardServer(1); + + // make sure the WAL syncs + Thread.sleep(TimeUnit.SECONDS.toMillis(1)); + + // This should block until shards have failed over + client.shardServerLayout(tableName); + + assertEquals("We should have lost a node.", 2, client.shardServerList(BlurConstants.DEFAULT).size()); + assertEquals(numberOfDocs, client.query(tableName, blurQuery).getTotalResults()); + + miniCluster.startShards(1, true, externalProcesses); + Thread.sleep(TimeUnit.SECONDS.toMillis(1)); + + assertEquals("We should have the cluster back where we started.", 3, client.shardServerList(BlurConstants.DEFAULT) + .size()); + } + + @Test + public void testTrucateRaceCondition() throws BlurException, TException, IOException, InterruptedException { + String tableName = "testTrucateRaceCondition"; + createTable(tableName); + loadTable(tableName); + List<Connection> connections = BlurClientManager.getConnections(miniCluster.getControllerConnectionStr()); + Iface client1 = BlurClient.getClient(connections.get(0)); + Iface client2 = BlurClient.getClient(connections.get(1)); + TableDescriptor describe = client1.describe(tableName); + setupTableProperties(describe); + client1.disableTable(tableName); + client1.removeTable(tableName, true); + client1.createTable(describe); + postTableCreate(describe, client1); + + String rowId = UUID.randomUUID().toString(); + RecordMutation mutation = BlurThriftHelper.newRecordMutation("test", rowId, + BlurThriftHelper.newColumn("test", "value"), BlurThriftHelper.newColumn("facetFixed", "test")); + RowMutation rowMutation = BlurThriftHelper.newRowMutation(tableName, rowId, mutation); + client2.mutate(mutate(rowMutation)); + } + + @Test + public void testTableRemovalWithFieldDefsV1() throws BlurException, TException, IOException, InterruptedException { + String tableName = "testTableRemovalWithFieldDefsV1"; + createTable(tableName); + loadTable(tableName); + + String family = "testTableRemovalWithFieldDefs-fam"; + + List<Connection> connections = BlurClientManager.getConnections(miniCluster.getControllerConnectionStr()); + Iface client1 = BlurClient.getClient(connections.get(0)); + ColumnDefinition columnDefinition = new ColumnDefinition(); + columnDefinition.setColumnName("col"); + columnDefinition.setFamily(family); + columnDefinition.setFieldLessIndexed(false); + columnDefinition.setFieldType("string"); + columnDefinition.setSortable(false); + columnDefinition.setSubColumnName(null); + assertTrue(client1.addColumnDefinition(tableName, columnDefinition)); + + client1.disableTable(tableName); + client1.removeTable(tableName, true); + + createTable(tableName); + + Iface client2 = BlurClient.getClient(connections.get(1)); + assertFamilyIsNotPresent(tableName, client2, family); + + List<String> shardClusterList = client2.shardClusterList(); + + for (String cluster : shardClusterList) { + List<String> shardServerList = client2.shardServerList(cluster); + for (String shardServer : shardServerList) { + Iface client = BlurClient.getClient(shardServer); + assertFamilyIsNotPresent(tableName, client, family); + } + } + + } + + @Test + public void testTableRemovalWithFieldDefsV2() throws BlurException, TException, IOException, InterruptedException { + String tableName = "testTableRemovalWithFieldDefsV2"; + createTable(tableName); + loadTable(tableName); + + Iface client = getClient(); + Schema expectedSchema = client.schema(tableName); + Set<String> expectedFiles = new TreeSet<String>(); + TableDescriptor describe = client.describe(tableName); + + { + Path path = new Path(describe.getTableUri()); + FileSystem fileSystem = path.getFileSystem(new Configuration()); + FileStatus[] listStatus = fileSystem.listStatus(new Path(path, "types")); + for (FileStatus fileStatus : listStatus) { + System.out.println("Expected " + fileStatus.getPath()); + expectedFiles.add(fileStatus.getPath().toString()); + } + } + + client.disableTable(tableName); + client.removeTable(tableName, true); + + createTable(tableName); + loadTable(tableName); + Schema actualSchema = client.schema(tableName); + + assertEquals(expectedSchema.getTable(), actualSchema.getTable()); + Map<String, Map<String, ColumnDefinition>> expectedFamilies = expectedSchema.getFamilies(); + Map<String, Map<String, ColumnDefinition>> actualFamilies = actualSchema.getFamilies(); + assertEquals(new TreeSet<String>(expectedFamilies.keySet()), new TreeSet<String>(actualFamilies.keySet())); + + for (String family : expectedFamilies.keySet()) { + Map<String, ColumnDefinition> expectedColDefMap = new TreeMap<String, ColumnDefinition>( + expectedFamilies.get(family)); + Map<String, ColumnDefinition> actualColDefMap = new TreeMap<String, ColumnDefinition>(actualFamilies.get(family)); + assertEquals(expectedColDefMap, actualColDefMap); + } + + System.out.println(expectedSchema); + + Set<String> actualFiles = new TreeSet<String>(); + { + Path path = new Path(describe.getTableUri()); + FileSystem fileSystem = path.getFileSystem(new Configuration()); + FileStatus[] listStatus = fileSystem.listStatus(new Path(path, "types")); + for (FileStatus fileStatus : listStatus) { + System.out.println("Actual " + fileStatus.getPath()); + actualFiles.add(fileStatus.getPath().toString()); + } + } + assertEquals(expectedFiles, actualFiles); + } + + private void assertFamilyIsNotPresent(String tableName, Iface client, String family) throws BlurException, TException { + Schema schema = client.schema(tableName); + Map<String, Map<String, ColumnDefinition>> families = schema.getFamilies(); + assertNull(families.get(family)); + } + + private void assertRowResults(BlurResults results) { + for (BlurResult result : results.getResults()) { + assertNull(result.locationId); + assertNull(result.fetchResult.recordResult); + assertNull(result.fetchResult.rowResult.row.records); + assertNotNull(result.fetchResult.rowResult.row.id); + } + } + + private void assertRecordResults(BlurResults results) { + for (BlurResult result : results.getResults()) { + assertNull(result.locationId); + assertNotNull(result.fetchResult.recordResult); + assertNotNull(result.fetchResult.recordResult.rowid); + assertNotNull(result.fetchResult.recordResult.record.recordId); + assertNotNull(result.fetchResult.recordResult.record.family); + assertNull("Not null [" + result.fetchResult.recordResult.record.columns + "]", + result.fetchResult.recordResult.record.columns); + assertNull(result.fetchResult.rowResult); + } + } + + @Test + public void testCreateDisableAndRemoveTable() throws IOException, BlurException, TException { + Iface client = getClient(); + String tableName = UUID.randomUUID().toString(); + TableDescriptor tableDescriptor = new TableDescriptor(); + tableDescriptor.setName(tableName); + tableDescriptor.setShardCount(5); + tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/" + tableName); + setupTableProperties(tableDescriptor); + for (int i = 0; i < 3; i++) { + client.createTable(tableDescriptor); + postTableCreate(tableDescriptor, client); + client.disableTable(tableName); + client.removeTable(tableName, true); + } + + assertFalse(client.tableList().contains(tableName)); + + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java new file mode 100644 index 0000000..1504f01 --- /dev/null +++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestNoSecurity.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.thrift; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.blur.thrift.generated.Blur.Iface; +import org.apache.blur.thrift.generated.RowMutation; +import org.apache.blur.thrift.generated.TableDescriptor; +import org.apache.blur.user.User; + +public class BlurClusterTestNoSecurity extends BlurClusterTestBase { + + @Override + protected void setupTableProperties(TableDescriptor tableDescriptor) { + // do nothing + } + + @Override + protected RowMutation mutate(RowMutation rowMutation) { + return rowMutation; + } + + @Override + protected void postTableCreate(TableDescriptor tableDescriptor, Iface client) { + + } + + @Override + protected User getUser() { + return null; + } + + @Override + protected Map<String, String> getUserAttributes() { + return new HashMap<String, String>(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestSecurity.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestSecurity.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestSecurity.java new file mode 100644 index 0000000..b669406 --- /dev/null +++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestSecurity.java @@ -0,0 +1,83 @@ +/** +ing ok * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.thrift; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thrift.generated.Blur.Iface; +import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.thrift.generated.Column; +import org.apache.blur.thrift.generated.ColumnDefinition; +import org.apache.blur.thrift.generated.Record; +import org.apache.blur.thrift.generated.RecordMutation; +import org.apache.blur.thrift.generated.RowMutation; +import org.apache.blur.thrift.generated.TableDescriptor; +import org.apache.blur.user.User; +import org.apache.blur.utils.BlurConstants; + +public class BlurClusterTestSecurity extends BlurClusterTestBase { + + private static final String DISCOVER = "discover"; + private static final String READ = "read"; + + @Override + protected void setupTableProperties(TableDescriptor tableDescriptor) { + tableDescriptor.putToTableProperties(BlurConstants.BLUR_RECORD_SECURITY, "true"); + } + + @Override + protected RowMutation mutate(RowMutation rowMutation) { + List<RecordMutation> mutations = rowMutation.getRecordMutations(); + for (RecordMutation mutation : mutations) { + Record record = mutation.getRecord(); + record.addToColumns(new Column("acl-read", READ)); + record.addToColumns(new Column("acl-discover", DISCOVER)); + } + return rowMutation; + } + + @Override + protected void postTableCreate(TableDescriptor tableDescriptor, Iface client) { + String name = tableDescriptor.getName(); + try { + client.addColumnDefinition(name, new ColumnDefinition("test", "acl-read", null, false, "acl-read", null, false)); + client.addColumnDefinition(name, new ColumnDefinition("test", "acl-discover", null, false, "acl-discover", null, + false)); + } catch (BlurException e) { + throw new RuntimeException(e); + } catch (TException e) { + throw new RuntimeException(e); + } + } + + @Override + protected User getUser() { + return new User("testuser", getUserAttributes()); + } + + @Override + protected Map<String, String> getUserAttributes() { + Map<String, String> attributes = new HashMap<String, String>(); + attributes.put(BlurConstants.ACL_READ, READ); + attributes.put(BlurConstants.ACL_DISCOVER, DISCOVER); + return attributes; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-query/pom.xml ---------------------------------------------------------------------- diff --git a/blur-query/pom.xml b/blur-query/pom.xml index 8be6d32..03eb6f3 100644 --- a/blur-query/pom.xml +++ b/blur-query/pom.xml @@ -63,11 +63,6 @@ <version>${lucene.version}</version> </dependency> <dependency> - <groupId>com.github.amccurry</groupId> - <artifactId>lucene-document-security</artifactId> - <version>0.1.0</version> - </dependency> - <dependency> <groupId>commons-cli</groupId> <artifactId>commons-cli</artifactId> <version>${commons-cli.version}</version> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-query/src/main/java/org/apache/blur/lucene/search/IndexSearcherCloseable.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/IndexSearcherCloseable.java b/blur-query/src/main/java/org/apache/blur/lucene/search/IndexSearcherCloseable.java new file mode 100644 index 0000000..31d4004 --- /dev/null +++ b/blur-query/src/main/java/org/apache/blur/lucene/search/IndexSearcherCloseable.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.search; + +import java.io.Closeable; +import java.io.IOException; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.similarities.Similarity; + +public interface IndexSearcherCloseable extends Closeable { + + IndexReader getIndexReader(); + + TopDocs search(Query query, int i) throws IOException; + + Query rewrite(Query query) throws IOException; + + void search(Query query, Collector collector) throws IOException; + + void setSimilarity(Similarity similarity); + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-query/src/main/java/org/apache/blur/lucene/search/IndexSearcherCloseableUtil.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/IndexSearcherCloseableUtil.java b/blur-query/src/main/java/org/apache/blur/lucene/search/IndexSearcherCloseableUtil.java new file mode 100644 index 0000000..973a5ba --- /dev/null +++ b/blur-query/src/main/java/org/apache/blur/lucene/search/IndexSearcherCloseableUtil.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.search; + +import java.io.IOException; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.similarities.Similarity; + +public class IndexSearcherCloseableUtil { + + public static IndexSearcherCloseable wrap(final IndexSearcher searcher) { + return new IndexSearcherCloseable() { + + @Override + public void close() throws IOException { + + } + + @Override + public void setSimilarity(Similarity similarity) { + searcher.setSimilarity(similarity); + } + + @Override + public void search(Query query, Collector collector) throws IOException { + searcher.search(query, collector); + } + + @Override + public TopDocs search(Query query, int n) throws IOException { + return searcher.search(query, n); + } + + @Override + public Query rewrite(Query query) throws IOException { + return searcher.rewrite(query); + } + + @Override + public IndexReader getIndexReader() { + return searcher.getIndexReader(); + } + }; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0b065b16/blur-query/src/main/java/org/apache/blur/lucene/search/IterablePaging.java ---------------------------------------------------------------------- diff --git a/blur-query/src/main/java/org/apache/blur/lucene/search/IterablePaging.java b/blur-query/src/main/java/org/apache/blur/lucene/search/IterablePaging.java index 3bb2d74..8089346 100644 --- a/blur-query/src/main/java/org/apache/blur/lucene/search/IterablePaging.java +++ b/blur-query/src/main/java/org/apache/blur/lucene/search/IterablePaging.java @@ -34,7 +34,6 @@ import org.apache.blur.utils.BlurIterable; import org.apache.blur.utils.BlurIterator; import org.apache.lucene.search.Collector; import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; @@ -53,7 +52,7 @@ public class IterablePaging implements BlurIterable<ScoreDoc, BlurException> { private static final boolean DISABLED = true; private final DeepPagingCache _deepPagingCache; - private final IndexSearcher _searcher; + private final IndexSearcherCloseable _searcher; private final Query _query; private final AtomicBoolean _running; private final int _numHitsToCollect; @@ -66,7 +65,7 @@ public class IterablePaging implements BlurIterable<ScoreDoc, BlurException> { private int skipTo; private int gather = -1; - public IterablePaging(AtomicBoolean running, IndexSearcher searcher, Query query, int numHitsToCollect, + public IterablePaging(AtomicBoolean running, IndexSearcherCloseable searcher, Query query, int numHitsToCollect, TotalHitsRef totalHitsRef, ProgressRef progressRef, boolean runSlow, Sort sort, DeepPagingCache deepPagingCache) throws BlurException { _deepPagingCache = deepPagingCache;
