Repository: incubator-sentry Updated Branches: refs/heads/master 0e07e9bf9 -> 574a6855d
SENTRY-159: Convert AbstractSolrSentryTestBase to use MiniSolrCloudCluster rather than Lucene test hierarchy (Gregory Chanan via Vamsee Yarlagadda) Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/574a6855 Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/574a6855 Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/574a6855 Branch: refs/heads/master Commit: 574a6855dbfdce3ab7934ffc0b1d8a3138def674 Parents: 0e07e9b Author: Vamsee <[email protected]> Authored: Tue Apr 29 13:17:00 2014 -0700 Committer: Vamsee <[email protected]> Committed: Tue Apr 29 13:17:00 2014 -0700 ---------------------------------------------------------------------- pom.xml | 2 +- .../e2e/solr/AbstractSolrSentryTestBase.java | 142 +++++++++++++------ .../e2e/solr/TestCollAdminCoreOperations.java | 21 ++- .../tests/e2e/solr/TestQueryOperations.java | 15 +- .../tests/e2e/solr/TestUpdateOperations.java | 23 ++- 5 files changed, 126 insertions(+), 77 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/574a6855/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index c52f7ad..373c25b 100644 --- a/pom.xml +++ b/pom.xml @@ -78,7 +78,7 @@ limitations under the License. <mockito.version>1.8.5</mockito.version> <shiro.version>1.2.1</shiro.version> <slf4j.version>1.6.1</slf4j.version> - <solr.version>4.7.0</solr.version> + <solr.version>4.8.0</solr.version> <solr.sentry.handlers.version>4.4.0-cdh5.1.0-SNAPSHOT</solr.sentry.handlers.version> <zookeeper.version>3.4.5-cdh5.1.0-SNAPSHOT</zookeeper.version> </properties> http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/574a6855/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java index bc36967..d58f3b9 100644 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java @@ -20,15 +20,22 @@ import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.util.Comparator; +import java.util.Map; import java.util.Random; import java.util.SortedMap; import java.util.TreeMap; +import com.google.common.io.Files; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import org.apache.commons.io.FileUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.impl.CloudSolrServer; @@ -37,10 +44,15 @@ import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.cloud.AbstractFullDistribZkTestBase; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.ZkController; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.ModifiableSolrParams; @@ -53,14 +65,17 @@ import org.junit.BeforeClass; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class AbstractSolrSentryTestBase extends AbstractFullDistribZkTestBase { +public class AbstractSolrSentryTestBase { private static final Logger LOG = LoggerFactory.getLogger(AbstractSolrSentryTestBase.class); protected static final String SENTRY_ERROR_MSG = "SentrySolrAuthorizationException"; private static MiniDFSCluster dfsCluster; + private static MiniSolrCloudCluster miniSolrCloudCluster; private static SortedMap<Class, String> extraRequestFilters; protected static final String ADMIN_USER = "admin"; protected static final String ALL_DOCS = "*:*"; protected static final Random RANDOM = new Random(); + protected static final String RESOURCES_DIR = "target" + File.separator + "test-classes" + File.separator + "solr"; + private static final int NUM_SERVERS = 4; private static void addPropertyToSentry(StringBuilder builder, String name, String value) { builder.append("<property>\n"); @@ -72,7 +87,7 @@ public abstract class AbstractSolrSentryTestBase extends AbstractFullDistribZkTe public static File setupSentry() throws Exception { File sentrySite = File.createTempFile("sentry-site", "xml"); sentrySite.deleteOnExit(); - File authProviderDir = new File(SolrTestCaseJ4.TEST_HOME(), "sentry"); + File authProviderDir = new File(RESOURCES_DIR, "sentry"); String authProviderName = "test-authz-provider.ini"; FileSystem clusterFs = dfsCluster.getFileSystem(); clusterFs.copyFromLocalFile(false, @@ -94,7 +109,7 @@ public abstract class AbstractSolrSentryTestBase extends AbstractFullDistribZkTe @BeforeClass public static void beforeTestSimpleSolrEndToEnd() throws Exception { - dfsCluster = HdfsTestUtil.setupClass(new File(TEMP_DIR, + dfsCluster = HdfsTestUtil.setupClass(new File(Files.createTempDir(), AbstractSolrSentryTestBase.class.getName() + "_" + System.currentTimeMillis()).getAbsolutePath()); File sentrySite = setupSentry(); @@ -111,6 +126,9 @@ public abstract class AbstractSolrSentryTestBase extends AbstractFullDistribZkTe } }); extraRequestFilters.put(ModifiableUserAuthenticationFilter.class, "*"); + File solrXml = new File(RESOURCES_DIR, "solr-no-core.xml"); + miniSolrCloudCluster = new MiniSolrCloudCluster(NUM_SERVERS, null, solrXml, + null, extraRequestFilters); } @AfterClass @@ -120,42 +138,19 @@ public abstract class AbstractSolrSentryTestBase extends AbstractFullDistribZkTe System.clearProperty("solr.authorization.sentry.site"); dfsCluster = null; extraRequestFilters = null; + miniSolrCloudCluster.shutdown(); } @Before public void setupBeforeTest() throws Exception { - System.setProperty("numShards", Integer.toString(sliceCount)); System.setProperty("solr.xml.persist", "true"); - super.setUp(); } @After public void tearDown() throws Exception { - super.tearDown(); - System.clearProperty("numShards"); System.clearProperty("solr.xml.persist"); } - @Override - protected String getDataDir(String dataDir) throws IOException { - return HdfsTestUtil.getDataDir(dfsCluster, dataDir); - } - - @Override - protected String getSolrXml() { - return "solr-no-core.xml"; - } - - @Override - protected String getCloudSolrConfig() { - return "solrconfig.xml"; - } - - @Override - public SortedMap<Class,String> getExtraRequestFilters() { - return extraRequestFilters; - } - /** * Set the proper user in the Solr authentication filter * @param solrUser @@ -410,12 +405,12 @@ public abstract class AbstractSolrSentryTestBase extends AbstractFullDistribZkTe try { setAuthenticationUser(solrUserName); QueryRequest request = populateCollectionAdminParams(adminOp, collectionName); - SolrServer solrServer = createNewSolrServer("", getBaseUrl((HttpSolrServer) clients.get(0))); + CloudSolrServer solrServer = createNewCloudSolrServer(); try { NamedList<Object> result = solrServer.request(request); if (adminOp.compareTo(CollectionAction.CREATE) == 0) { // Wait for collection creation to complete. - waitForRecoveriesToFinish(collectionName, false); + waitForRecoveriesToFinish(collectionName, solrServer, false); } } finally { solrServer.shutdown(); @@ -441,12 +436,12 @@ public abstract class AbstractSolrSentryTestBase extends AbstractFullDistribZkTe setAuthenticationUser(solrUserName); try { QueryRequest request = populateCollectionAdminParams(adminOp, collectionName); - SolrServer solrServer = createNewSolrServer("", getBaseUrl((HttpSolrServer) clients.get(0))); + CloudSolrServer solrServer = createNewCloudSolrServer(); try { NamedList<Object> result = solrServer.request(request); if (adminOp.compareTo(CollectionAction.CREATE) == 0) { // Wait for collection creation to complete. - waitForRecoveriesToFinish(collectionName, false); + waitForRecoveriesToFinish(collectionName, solrServer, false); } } finally { solrServer.shutdown(); @@ -606,8 +601,8 @@ public abstract class AbstractSolrSentryTestBase extends AbstractFullDistribZkTe * @throws MalformedURLException */ protected CloudSolrServer getCloudSolrServer(String collectionName) throws MalformedURLException { - CloudSolrServer cloudSolrServer = new CloudSolrServer(zkServer.getZkAddress(), - random().nextBoolean()); + CloudSolrServer cloudSolrServer = new CloudSolrServer(miniSolrCloudCluster.getZkServer().getZkAddress(), + RANDOM.nextBoolean()); cloudSolrServer.setDefaultCollection(collectionName); cloudSolrServer.connect(); return cloudSolrServer; @@ -680,19 +675,80 @@ public abstract class AbstractSolrSentryTestBase extends AbstractFullDistribZkTe verifyUpdatePass(ADMIN_USER, collectionName, solrInputDoc); } - /** - * Subclasses can override this to change a test's solr home - * (default is in test-files) - */ - public String getSolrHome() { - return SolrTestCaseJ4.TEST_HOME(); - } - protected void uploadConfigDirToZk(String collectionConfigDir) throws Exception { SolrDispatchFilter dispatchFilter = - (SolrDispatchFilter) jettys.get(0).getDispatchFilter().getFilter(); + (SolrDispatchFilter) miniSolrCloudCluster.getJettySolrRunners().get(0).getDispatchFilter().getFilter(); ZkController zkController = dispatchFilter.getCores().getZkController(); // conf1 is the config used by AbstractFullDistribZkTestBase zkController.uploadConfigDir(new File(collectionConfigDir), "conf1"); } + + protected CloudSolrServer createNewCloudSolrServer() throws Exception { + CloudSolrServer css = new CloudSolrServer(miniSolrCloudCluster.getZkServer().getZkAddress()); + css.connect(); + return css; + } + + protected static void waitForRecoveriesToFinish(String collection, + CloudSolrServer solrServer, + boolean verbose) throws Exception { + waitForRecoveriesToFinish(collection, solrServer, verbose, true, 60); + } + + protected static void waitForRecoveriesToFinish(String collection, + CloudSolrServer solrServer, + boolean verbose, + boolean failOnTimeout, + int timeoutSeconds) throws Exception { + LOG.info("Entering solr wait with timeout " + timeoutSeconds); + ZkStateReader zkStateReader = solrServer.getZkStateReader(); + try { + boolean cont = true; + int cnt = 0; + + while (cont) { + if (verbose) LOG.debug("-"); + boolean sawLiveRecovering = false; + zkStateReader.updateClusterState(true); + ClusterState clusterState = zkStateReader.getClusterState(); + Map<String, Slice> slices = clusterState.getSlicesMap(collection); + assertNotNull("Could not find collection:" + collection, slices); + for (Map.Entry<String, Slice> entry : slices.entrySet()) { + Map<String, Replica> shards = entry.getValue().getReplicasMap(); + for (Map.Entry<String, Replica> shard : shards.entrySet()) { + if (verbose) LOG.debug("rstate:" + + shard.getValue().getStr(ZkStateReader.STATE_PROP) + " live:" + + clusterState.liveNodesContain(shard.getValue().getNodeName())); + String state = shard.getValue().getStr(ZkStateReader.STATE_PROP); + if ((state.equals(ZkStateReader.RECOVERING) + || state.equals(ZkStateReader.SYNC) || state + .equals(ZkStateReader.DOWN)) + && clusterState.liveNodesContain(shard.getValue().getStr( + ZkStateReader.NODE_NAME_PROP))) { + sawLiveRecovering = true; + } + } + } + if (!sawLiveRecovering || cnt == timeoutSeconds) { + if (!sawLiveRecovering) { + if (verbose) LOG.debug("no one is recovering"); + } else { + if (verbose) LOG.debug("Gave up waiting for recovery to finish.."); + if (failOnTimeout) { + fail("There are still nodes recovering - waited for " + + timeoutSeconds + " seconds"); + // won't get here + return; + } + } + cont = false; + } else { + Thread.sleep(1000); + } + cnt++; + } + } finally { + LOG.info("Exiting solr wait"); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/574a6855/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestCollAdminCoreOperations.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestCollAdminCoreOperations.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestCollAdminCoreOperations.java index 8509497..b0d6db1 100644 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestCollAdminCoreOperations.java +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestCollAdminCoreOperations.java @@ -16,6 +16,9 @@ */ package org.apache.sentry.tests.e2e.solr; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.File; import java.io.PrintWriter; import java.io.StringWriter; @@ -26,24 +29,18 @@ import java.util.List; import java.util.Random; import org.apache.solr.common.params.CollectionParams.CollectionAction; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope; +import org.junit.Test; +import static org.junit.Assert.assertEquals; -@ThreadLeakScope(Scope.NONE) // hdfs client currently leaks thread(s) public class TestCollAdminCoreOperations extends AbstractSolrSentryTestBase { - // Necessary until we pull in a solr version with SOLR-5771 - static { - ALLOW_SSL = false; - } private static final Logger LOG = LoggerFactory .getLogger(TestCollAdminCoreOperations.class); private static final String ADMIN_COLLECTION_NAME = "admin"; private static final String TEST_COLLECTION_NAME = "sentryCollection"; private static final List<Boolean> BOOLEAN_VALUES = Arrays.asList(new Boolean[]{true, false}); + private static final String DEFAULT_COLLECTION = "collection1"; /** * Maximum number of combinations that will be tested by this class. @@ -55,8 +52,8 @@ public class TestCollAdminCoreOperations extends AbstractSolrSentryTestBase { */ private static int NUM_TESTS_TO_RUN = 15; - @Override - public void doTest() throws Exception { + @Test + public void testCollAdminCoreOperations() throws Exception { String maxTestsToRun = System.getProperty("sentry.solr.e2e.maxTestsToRun"); if (maxTestsToRun != null) { if (maxTestsToRun.compareToIgnoreCase("all") == 0) { @@ -78,7 +75,7 @@ public class TestCollAdminCoreOperations extends AbstractSolrSentryTestBase { ArrayList<String> testFailures = new ArrayList<String>(); // Upload configs to ZK - uploadConfigDirToZk(getSolrHome() + File.separator + DEFAULT_COLLECTION + uploadConfigDirToZk(RESOURCES_DIR + File.separator + DEFAULT_COLLECTION + File.separator + "conf"); for (boolean admin_query : BOOLEAN_VALUES) { for (boolean admin_update : BOOLEAN_VALUES) { http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/574a6855/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestQueryOperations.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestQueryOperations.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestQueryOperations.java index 6658560..c257175 100644 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestQueryOperations.java +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestQueryOperations.java @@ -30,22 +30,21 @@ import org.slf4j.LoggerFactory; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope; -@ThreadLeakScope(Scope.NONE) // hdfs client currently leaks thread(s) +import org.junit.Test; +import static org.junit.Assert.assertEquals; + public class TestQueryOperations extends AbstractSolrSentryTestBase { - // Necessary until we pull in a solr version with SOLR-5771 - static { - ALLOW_SSL = false; - } private static final Logger LOG = LoggerFactory .getLogger(TestQueryOperations.class); private static final String COLLECTION_NAME = "sentryCollection"; private static final List<Boolean> BOOLEAN_VALUES = Arrays.asList(new Boolean[]{true, false}); + private static final String DEFAULT_COLLECTION = "collection1"; - @Override - public void doTest() throws Exception { + @Test + public void testQueryOps() throws Exception { // Upload configs to ZK - uploadConfigDirToZk(getSolrHome() + File.separator + DEFAULT_COLLECTION + uploadConfigDirToZk(RESOURCES_DIR + File.separator + DEFAULT_COLLECTION + File.separator + "conf"); setupCollection(COLLECTION_NAME); ArrayList<String> testFailures = new ArrayList<String>(); http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/574a6855/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestUpdateOperations.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestUpdateOperations.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestUpdateOperations.java index d4855da..7163573 100644 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestUpdateOperations.java +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestUpdateOperations.java @@ -16,6 +16,12 @@ */ package org.apache.sentry.tests.e2e.solr; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; + import java.io.File; import java.io.PrintWriter; import java.io.StringWriter; @@ -24,28 +30,19 @@ import java.util.Arrays; import java.util.List; import org.apache.solr.common.SolrInputDocument; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope; - -@ThreadLeakScope(Scope.NONE) // hdfs client currently leaks thread(s) public class TestUpdateOperations extends AbstractSolrSentryTestBase { - // Necessary until we pull in a solr version with SOLR-5771 - static { - ALLOW_SSL = false; - } private static final Logger LOG = LoggerFactory .getLogger(TestUpdateOperations.class); private static final String COLLECTION_NAME = "sentryCollection"; private static final List<Boolean> BOOLEAN_VALUES = Arrays.asList(new Boolean[]{true, false}); + private static final String DEFAULT_COLLECTION = "collection1"; - @Override - public void doTest() throws Exception { + @Test + public void testUpdateOperations() throws Exception { // Upload configs to ZK - uploadConfigDirToZk(getSolrHome() + File.separator + DEFAULT_COLLECTION + uploadConfigDirToZk(RESOURCES_DIR + File.separator + DEFAULT_COLLECTION + File.separator + "conf"); setupCollection(COLLECTION_NAME); ArrayList<String> testFailures = new ArrayList<String>();
