[CDH-45122] Fix org.apache.sentry.tests.e2e.solr.TestCollAdminCoreOperations
Identified that randomizedtesting framework was not configured correctly for Solr/Sentry end-to-end tests. Hence updated AbstractSolrSentryTestBase to extend from SolrTestCaseJ4 which sets up the randomizedtesting framework appropriately. This uncovered number of other problems in the tests which required additional changes, - Added a separate base class for file-provider based tests. - The tests for Sentry service fail due to number of thread leaks in Sentry service. Added SentryServiceThreadLeakFilter to supress these for now. After these leaks are fixed, SentryServiceThreadLeakFilter.java should be removed. Change-Id: I71e40b1f1fc67814e6c3276954bc61cfe5d0f0d2 Reviewed-on: http://gerrit.sjc.cloudera.com:8080/17192 Tested-by: Jenkins User Reviewed-by: Kalyan Kumar Kalvagadda <[email protected]> Reviewed-on: http://gerrit.sjc.cloudera.com:8080/22691 Reviewed-by: Alexander Kolbasov <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/8841a023 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/8841a023 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/8841a023 Branch: refs/for/cdh5-1.5.1_ha Commit: 8841a02367679cff8ac6ff6e125268bb2a5ca5dc Parents: a0efa35 Author: Alexander Kolbasov <[email protected]> Authored: Mon May 15 21:47:38 2017 -0700 Committer: Alexander Kolbasov <[email protected]> Committed: Mon May 15 23:03:26 2017 -0700 ---------------------------------------------------------------------- pom.xml | 4 - .../e2e/solr/AbstractSolrSentryTestBase.java | 82 ++------------ .../AbstractSolrSentryTestWithFileProvider.java | 109 +++++++++++++++++++ .../e2e/solr/TestCollAdminCoreOperations.java | 3 +- .../tests/e2e/solr/TestDocLevelOperations.java | 8 +- .../tests/e2e/solr/TestQueryOperations.java | 6 +- .../sentry/tests/e2e/solr/TestRealTimeGet.java | 8 +- .../tests/e2e/solr/TestUpdateOperations.java | 5 +- .../AbstractSolrSentryTestWithDbProvider.java | 37 +++++++ .../SentryServiceThreadLeakFilter.java | 44 ++++++++ .../db/integration/TestSolrAdminOperations.java | 2 - 11 files changed, 208 insertions(+), 100 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/8841a023/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index f131d2d..a72434d 100644 --- a/pom.xml +++ b/pom.xml @@ -940,10 +940,6 @@ limitations under the License. <exclude>%regex[org.apache.sentry.tests.e2e.dbprovider.*.class]</exclude> <exclude>%regex[org.apache.sentry.tests.e2e.hive.TestPrivilegesAtFunctionScope.*.class]</exclude> <exclude>%regex[org.apache.sentry.tests.e2e.hdfs.TestHDFS.*.class]</exclude> - <!-- Add the solr test back when CDH-45122 is resolved. --> - <exclude>%regex[org.apache.sentry.tests.e2e.solr.*.class]</exclude> - <!-- Add the HMSNotificationLog test back when CDH-45447 is resolved. --> - <exclude>%regex[org.apache.sentry.tests.e2e.metastore.TestHMSNotificationLogUsingDBNotificationListener.*.class]</exclude> </excludes> </configuration> </plugin> http://git-wip-us.apache.org/repos/asf/sentry/blob/8841a023/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java index 3a2104a..da34c7c 100644 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java @@ -16,24 +16,16 @@ */ package org.apache.sentry.tests.e2e.solr; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; import java.net.URI; -import java.util.Comparator; import java.util.Iterator; import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; @@ -51,13 +43,12 @@ import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.util.EntityUtils; -import org.apache.sentry.binding.solr.HdfsTestUtil; +import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.impl.CloudSolrServer; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.util.ClientUtils; -import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; @@ -71,22 +62,18 @@ import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; -import org.apache.solr.servlet.SolrDispatchFilter; import org.junit.After; -import org.junit.AfterClass; import org.junit.Before; -import org.junit.BeforeClass; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.io.Files; - -public class AbstractSolrSentryTestBase { +/** + * This is a base class used by all the Solr Sentry-plugin unit tests. + * + */ +public abstract class AbstractSolrSentryTestBase extends SolrTestCaseJ4 { private static final Logger LOG = LoggerFactory.getLogger(AbstractSolrSentryTestBase.class); protected static final String SENTRY_ERROR_MSG = "SentrySolrAuthorizationException"; - protected static MiniDFSCluster dfsCluster; - protected static MiniSolrCloudCluster miniSolrCloudCluster; - protected static SortedMap<Class, String> extraRequestFilters; protected static final String ADMIN_USER = "admin"; protected static final String ALL_DOCS = "*:*"; protected static final Random RANDOM = new Random(); @@ -102,7 +89,7 @@ public class AbstractSolrSentryTestBase { builder.append("</property>\n"); } - public static File setupSentry() throws Exception { + public static File setupSentry(MiniDFSCluster dfsCluster) throws Exception { File sentrySite = File.createTempFile("sentry-site", "xml"); sentrySite.deleteOnExit(); File authProviderDir = new File(RESOURCES_DIR, "sentry"); @@ -125,40 +112,6 @@ public class AbstractSolrSentryTestBase { return sentrySite; } - @BeforeClass - public static void beforeTestSimpleSolrEndToEnd() throws Exception { - dfsCluster = HdfsTestUtil.setupClass(new File(Files.createTempDir(), - AbstractSolrSentryTestBase.class.getName() + "_" - + System.currentTimeMillis()).getAbsolutePath()); - File sentrySite = setupSentry(); - System.setProperty("solr.authorization.sentry.site", sentrySite.toURI().toURL().toString().substring("file:".length())); - System.setProperty("solr.hdfs.home", dfsCluster.getURI().toString() + "/solr"); - extraRequestFilters = new TreeMap<Class, String>(new Comparator<Class>() { - // There's only one class, make this as simple as possible - public int compare(Class o1, Class o2) { - return 0; - } - - public boolean equals(Object obj) { - return true; - } - }); - extraRequestFilters.put(ModifiableUserAuthenticationFilter.class, "*"); - File solrXml = new File(RESOURCES_DIR, "solr-no-core.xml"); - miniSolrCloudCluster = new MiniSolrCloudCluster(NUM_SERVERS, null, solrXml, - null, extraRequestFilters); - } - - @AfterClass - public static void teardownClass() throws Exception { - HdfsTestUtil.teardownClass(dfsCluster); - System.clearProperty("solr.hdfs.home"); - System.clearProperty("solr.authorization.sentry.site"); - dfsCluster = null; - extraRequestFilters = null; - miniSolrCloudCluster.shutdown(); - } - @Before public void setupBeforeTest() throws Exception { System.setProperty("solr.xml.persist", "true"); @@ -169,6 +122,7 @@ public class AbstractSolrSentryTestBase { @After public void tearDown() throws Exception { + super.tearDown(); System.clearProperty("solr.hdfs.blockcache.enabled"); System.clearProperty("solr.xml.persist"); } @@ -671,13 +625,7 @@ public class AbstractSolrSentryTestBase { * @return instance of CloudSolrServer * @throws MalformedURLException */ - protected CloudSolrServer getCloudSolrServer(String collectionName) throws MalformedURLException { - CloudSolrServer cloudSolrServer = new CloudSolrServer(miniSolrCloudCluster.getZkServer().getZkAddress(), - RANDOM.nextBoolean()); - cloudSolrServer.setDefaultCollection(collectionName); - cloudSolrServer.connect(); - return cloudSolrServer; - } + protected abstract CloudSolrServer getCloudSolrServer(String collectionName) throws MalformedURLException; /** * Function to create a solr collection with the name passed as parameter @@ -746,11 +694,7 @@ public class AbstractSolrSentryTestBase { verifyUpdatePass(ADMIN_USER, collectionName, solrInputDoc); } - private ZkController getZkController() { - SolrDispatchFilter dispatchFilter = - (SolrDispatchFilter) miniSolrCloudCluster.getJettySolrRunners().get(0).getDispatchFilter().getFilter(); - return dispatchFilter.getCores().getZkController(); - } + protected abstract ZkController getZkController(); protected void uploadConfigDirToZk(String collectionConfigDir) throws Exception { uploadConfigDirToZk(collectionConfigDir, CONF_DIR_IN_ZK); @@ -771,11 +715,7 @@ public class AbstractSolrSentryTestBase { + confDirInZk + "/" + nameInZk, new File(file), false, true); } - protected CloudSolrServer createNewCloudSolrServer() throws Exception { - CloudSolrServer css = new CloudSolrServer(miniSolrCloudCluster.getZkServer().getZkAddress()); - css.connect(); - return css; - } + protected abstract CloudSolrServer createNewCloudSolrServer() throws Exception; /** * Make a raw http request to specific cluster node. Node is of the format http://git-wip-us.apache.org/repos/asf/sentry/blob/8841a023/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestWithFileProvider.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestWithFileProvider.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestWithFileProvider.java new file mode 100644 index 0000000..d97e3fc --- /dev/null +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestWithFileProvider.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.tests.e2e.solr; + +import java.io.File; +import java.net.MalformedURLException; +import java.util.Comparator; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.sentry.binding.solr.HdfsTestUtil; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.impl.CloudSolrServer; +import org.apache.solr.cloud.MiniSolrCloudCluster; +import org.apache.solr.cloud.ZkController; +import org.apache.solr.servlet.SolrDispatchFilter; +import org.apache.solr.util.BadHdfsThreadsFilter; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import com.google.common.io.Files; + +/** + * This is a base class for all unit tests related to Solr Sentry file-provider. + * + */ +@ThreadLeakFilters(defaultFilters = true, filters = { + BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) +}) [email protected] +public class AbstractSolrSentryTestWithFileProvider extends AbstractSolrSentryTestBase { + protected static SortedMap<Class, String> extraRequestFilters; + protected static MiniDFSCluster dfsCluster; + protected static MiniSolrCloudCluster miniSolrCloudCluster; + + @BeforeClass + public static void beforeTestSimpleSolrEndToEnd() throws Exception { + dfsCluster = HdfsTestUtil.setupClass(new File(Files.createTempDir(), + AbstractSolrSentryTestBase.class.getName() + "_" + + System.currentTimeMillis()).getAbsolutePath()); + File sentrySite = setupSentry(dfsCluster); + System.setProperty("solr.authorization.sentry.site", sentrySite.toURI().toURL().toString().substring("file:".length())); + System.setProperty("solr.hdfs.home", dfsCluster.getURI().toString() + "/solr"); + extraRequestFilters = new TreeMap<Class, String>(new Comparator<Class>() { + // There's only one class, make this as simple as possible + public int compare(Class o1, Class o2) { + return 0; + } + + public boolean equals(Object obj) { + return true; + } + }); + extraRequestFilters.put(ModifiableUserAuthenticationFilter.class, "*"); + File solrXml = new File(RESOURCES_DIR, "solr-no-core.xml"); + miniSolrCloudCluster = new MiniSolrCloudCluster(NUM_SERVERS, null, solrXml, + null, extraRequestFilters); + } + + @AfterClass + public static void teardownClass() throws Exception { + HdfsTestUtil.teardownClass(dfsCluster); + System.clearProperty("solr.hdfs.home"); + System.clearProperty("solr.authorization.sentry.site"); + dfsCluster = null; + extraRequestFilters = null; + if (miniSolrCloudCluster != null) { + miniSolrCloudCluster.shutdown(); + miniSolrCloudCluster = null; + } + } + + protected CloudSolrServer getCloudSolrServer(String collectionName) throws MalformedURLException { + CloudSolrServer cloudSolrServer = new CloudSolrServer(miniSolrCloudCluster.getZkServer().getZkAddress(), + RANDOM.nextBoolean()); + cloudSolrServer.setDefaultCollection(collectionName); + cloudSolrServer.connect(); + return cloudSolrServer; + } + + protected CloudSolrServer createNewCloudSolrServer() throws Exception { + CloudSolrServer css = new CloudSolrServer(miniSolrCloudCluster.getZkServer().getZkAddress()); + css.connect(); + return css; + } + + protected ZkController getZkController() { + SolrDispatchFilter dispatchFilter = + (SolrDispatchFilter) miniSolrCloudCluster.getJettySolrRunners().get(0).getDispatchFilter().getFilter(); + return dispatchFilter.getCores().getZkController(); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/8841a023/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestCollAdminCoreOperations.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestCollAdminCoreOperations.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestCollAdminCoreOperations.java index b0d6db1..d48f3d5 100644 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestCollAdminCoreOperations.java +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestCollAdminCoreOperations.java @@ -31,9 +31,8 @@ import java.util.Random; import org.apache.solr.common.params.CollectionParams.CollectionAction; import org.junit.Test; -import static org.junit.Assert.assertEquals; -public class TestCollAdminCoreOperations extends AbstractSolrSentryTestBase { +public class TestCollAdminCoreOperations extends AbstractSolrSentryTestWithFileProvider { private static final Logger LOG = LoggerFactory .getLogger(TestCollAdminCoreOperations.class); http://git-wip-us.apache.org/repos/asf/sentry/blob/8841a023/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java index 46399df..ebd0c6a 100644 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java @@ -20,8 +20,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.junit.After; import org.junit.Before; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.impl.CloudSolrServer; @@ -33,8 +31,6 @@ import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.NamedList; - import java.io.File; import java.net.URLEncoder; import java.util.ArrayList; @@ -45,7 +41,7 @@ import org.junit.Test; /** * Test the document-level security features */ -public class TestDocLevelOperations extends AbstractSolrSentryTestBase { +public class TestDocLevelOperations extends AbstractSolrSentryTestWithFileProvider { private static final Logger LOG = LoggerFactory .getLogger(TestDocLevelOperations.class); private static final String AUTH_FIELD = "sentry_auth"; @@ -135,7 +131,7 @@ public class TestDocLevelOperations extends AbstractSolrSentryTestBase { String id = doc.getFieldValue("id").toString(); assertEquals(1, Long.valueOf(id) % 2); } - + if (checkNonAdminUsers) { // as junit -- should get half the documents setAuthenticationUser("junit"); http://git-wip-us.apache.org/repos/asf/sentry/blob/8841a023/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestQueryOperations.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestQueryOperations.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestQueryOperations.java index c257175..827fd26 100644 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestQueryOperations.java +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestQueryOperations.java @@ -27,13 +27,9 @@ import org.apache.solr.common.SolrInputDocument; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope; - import org.junit.Test; -import static org.junit.Assert.assertEquals; -public class TestQueryOperations extends AbstractSolrSentryTestBase { +public class TestQueryOperations extends AbstractSolrSentryTestWithFileProvider { private static final Logger LOG = LoggerFactory .getLogger(TestQueryOperations.class); http://git-wip-us.apache.org/repos/asf/sentry/blob/8841a023/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestRealTimeGet.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestRealTimeGet.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestRealTimeGet.java index 0d25562..92f6ed0 100644 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestRealTimeGet.java +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestRealTimeGet.java @@ -32,10 +32,6 @@ import org.slf4j.LoggerFactory; import org.junit.After; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; @@ -47,7 +43,7 @@ import java.util.List; import java.util.Random; import java.util.Set; -public class TestRealTimeGet extends AbstractSolrSentryTestBase { +public class TestRealTimeGet extends AbstractSolrSentryTestWithFileProvider { private static final Logger LOG = LoggerFactory .getLogger(TestRealTimeGet.class); private static final String AUTH_FIELD = "sentry_auth"; @@ -292,7 +288,7 @@ public class TestRealTimeGet extends AbstractSolrSentryTestBase { doc2.addField("id", "2"); doc2.addField("description", "description2"); doc2.addField(AUTH_FIELD, "admin_role"); - + SolrInputDocument doc3 = new SolrInputDocument(); doc3.addField("id", "3"); doc3.addField("description", "description3"); http://git-wip-us.apache.org/repos/asf/sentry/blob/8841a023/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestUpdateOperations.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestUpdateOperations.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestUpdateOperations.java index 2b246b5..cdd7e2a 100644 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestUpdateOperations.java +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestUpdateOperations.java @@ -20,9 +20,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase; import org.apache.solr.client.solrj.impl.CloudSolrServer; @@ -38,7 +35,7 @@ import java.util.List; import org.apache.solr.common.SolrInputDocument; -public class TestUpdateOperations extends AbstractSolrSentryTestBase { +public class TestUpdateOperations extends AbstractSolrSentryTestWithFileProvider { private static final Logger LOG = LoggerFactory .getLogger(TestUpdateOperations.class); http://git-wip-us.apache.org/repos/asf/sentry/blob/8841a023/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/db/integration/AbstractSolrSentryTestWithDbProvider.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/db/integration/AbstractSolrSentryTestWithDbProvider.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/db/integration/AbstractSolrSentryTestWithDbProvider.java index 60fada5..bd961f7 100644 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/db/integration/AbstractSolrSentryTestWithDbProvider.java +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/db/integration/AbstractSolrSentryTestWithDbProvider.java @@ -22,8 +22,10 @@ import static org.apache.sentry.core.model.search.SearchModelAuthorizable.Author import java.io.File; import java.io.FileOutputStream; +import java.net.MalformedURLException; import java.util.Comparator; import java.util.List; +import java.util.SortedMap; import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.TimeoutException; @@ -33,6 +35,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.sentry.core.common.exception.SentryUserException; @@ -54,12 +57,18 @@ import org.apache.sentry.service.thrift.ServiceConstants.ClientConfig; import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig; import org.apache.sentry.tests.e2e.solr.AbstractSolrSentryTestBase; import org.apache.sentry.tests.e2e.solr.ModifiableUserAuthenticationFilter; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.impl.CloudSolrServer; import org.apache.solr.cloud.MiniSolrCloudCluster; +import org.apache.solr.cloud.ZkController; +import org.apache.solr.servlet.SolrDispatchFilter; +import org.apache.solr.util.BadHdfsThreadsFilter; import org.junit.AfterClass; import org.junit.BeforeClass; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -67,6 +76,11 @@ import com.google.common.collect.Sets; * This class used to test the Solr integration with DB store. * It will set up a miniSolrCloud, miniHDFS and Sentry service in a JVM process. */ +@ThreadLeakFilters(defaultFilters = true, filters = { + BadHdfsThreadsFilter.class, // hdfs currently leaks thread(s) + SentryServiceThreadLeakFilter.class +}) [email protected] public class AbstractSolrSentryTestWithDbProvider extends AbstractSolrSentryTestBase{ private static final Logger LOGGER = LoggerFactory .getLogger(AbstractSolrSentryTestWithDbProvider.class); @@ -82,6 +96,9 @@ public class AbstractSolrSentryTestWithDbProvider extends AbstractSolrSentryTest protected static final Configuration conf = new Configuration(false); + protected static SortedMap<Class, String> extraRequestFilters; + protected static MiniDFSCluster dfsCluster; + protected static MiniSolrCloudCluster miniSolrCloudCluster; protected static SentryService server; protected static SentryGenericServiceClient client; @@ -316,4 +333,24 @@ public class AbstractSolrSentryTestWithDbProvider extends AbstractSolrSentryTest tPrivilege.setAuthorizables(authorizables); return tPrivilege; } + + protected CloudSolrServer getCloudSolrServer(String collectionName) throws MalformedURLException { + CloudSolrServer cloudSolrServer = new CloudSolrServer(miniSolrCloudCluster.getZkServer().getZkAddress(), + RANDOM.nextBoolean()); + cloudSolrServer.setDefaultCollection(collectionName); + cloudSolrServer.connect(); + return cloudSolrServer; + } + + protected CloudSolrServer createNewCloudSolrServer() throws Exception { + CloudSolrServer css = new CloudSolrServer(miniSolrCloudCluster.getZkServer().getZkAddress()); + css.connect(); + return css; + } + + protected ZkController getZkController() { + SolrDispatchFilter dispatchFilter = + (SolrDispatchFilter) miniSolrCloudCluster.getJettySolrRunners().get(0).getDispatchFilter().getFilter(); + return dispatchFilter.getCores().getZkController(); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sentry/blob/8841a023/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/db/integration/SentryServiceThreadLeakFilter.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/db/integration/SentryServiceThreadLeakFilter.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/db/integration/SentryServiceThreadLeakFilter.java new file mode 100644 index 0000000..482cdbe --- /dev/null +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/db/integration/SentryServiceThreadLeakFilter.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.tests.e2e.solr.db.integration; + +import java.util.Arrays; +import java.util.List; + +import com.carrotsearch.randomizedtesting.ThreadFilter; + + +/** + * This class suppresses the known thread leaks in the Sentry service. + * It should be removed after these thread leaks are fixed. + */ +public class SentryServiceThreadLeakFilter implements ThreadFilter { + private static final List<String> KNOWN_LEAKED_THREAD_NAMES = + Arrays.asList( + "BoneCP-keep-alive-scheduler", + "BoneCP-pool-watch-thread", + "SentryService-0", + "BoneCP-release-thread-helper-thread", + "com.google.common.base.internal.Finalizer", + "derby.rawStoreDaemon"); + + @Override + public boolean reject(Thread arg0) { + String tName = arg0.getName(); + return tName.startsWith("Timer") || KNOWN_LEAKED_THREAD_NAMES.contains(tName); + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/8841a023/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/db/integration/TestSolrAdminOperations.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/db/integration/TestSolrAdminOperations.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/db/integration/TestSolrAdminOperations.java index eb1e856..a1ece61 100644 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/db/integration/TestSolrAdminOperations.java +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/db/integration/TestSolrAdminOperations.java @@ -17,8 +17,6 @@ package org.apache.sentry.tests.e2e.solr.db.integration; -import static org.junit.Assert.assertTrue; - import java.io.File; import java.util.Arrays;
