http://git-wip-us.apache.org/repos/asf/sentry/blob/e62fa28d/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java deleted file mode 100644 index 7ddd1e2..0000000 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java +++ /dev/null @@ -1,923 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.tests.e2e.solr; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.util.Comparator; -import java.util.Iterator; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpHead; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.util.EntityUtils; -import org.apache.sentry.binding.solr.HdfsTestUtil; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.impl.CloudSolrServer; -import org.apache.solr.client.solrj.request.QueryRequest; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.client.solrj.util.ClientUtils; -import org.apache.solr.cloud.MiniSolrCloudCluster; -import org.apache.solr.cloud.ZkController; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams.CollectionAction; -import org.apache.solr.common.params.CoreAdminParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.servlet.SolrDispatchFilter; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.io.Files; - -public class AbstractSolrSentryTestBase { - private static final Logger LOG = LoggerFactory.getLogger(AbstractSolrSentryTestBase.class); - protected static final String SENTRY_ERROR_MSG = "SentrySolrAuthorizationException"; - protected static MiniDFSCluster dfsCluster; - protected static MiniSolrCloudCluster miniSolrCloudCluster; - protected static SortedMap<Class, String> extraRequestFilters; - protected static final String ADMIN_USER = "admin"; - protected static final String ALL_DOCS = "*:*"; - protected static final Random RANDOM = new Random(); - protected static final String RESOURCES_DIR = "target" + File.separator + "test-classes" + File.separator + "solr"; - protected static final String CONF_DIR_IN_ZK = "conf1"; - protected static final String DEFAULT_COLLECTION = "collection1"; - protected static final int NUM_SERVERS = 4; - - private static void addPropertyToSentry(StringBuilder builder, String name, String value) { - builder.append("<property>\n"); - builder.append("<name>").append(name).append("</name>\n"); - builder.append("<value>").append(value).append("</value>\n"); - builder.append("</property>\n"); - } - - public static File setupSentry() throws Exception { - File sentrySite = File.createTempFile("sentry-site", "xml"); - sentrySite.deleteOnExit(); - File authProviderDir = new File(RESOURCES_DIR, "sentry"); - String authProviderName = "test-authz-provider.ini"; - FileSystem clusterFs = dfsCluster.getFileSystem(); - clusterFs.copyFromLocalFile(false, - new Path(authProviderDir.toString(), authProviderName), - new Path(authProviderName)); - - // need to write sentry-site at execution time because we don't know - // the location of sentry.solr.provider.resource beforehand - StringBuilder sentrySiteData = new StringBuilder(); - sentrySiteData.append("<configuration>\n"); - addPropertyToSentry(sentrySiteData, "sentry.provider", - "org.apache.sentry.provider.file.LocalGroupResourceAuthorizationProvider"); - addPropertyToSentry(sentrySiteData, "sentry.solr.provider.resource", - clusterFs.getWorkingDirectory() + File.separator + authProviderName); - sentrySiteData.append("</configuration>\n"); - FileUtils.writeStringToFile(sentrySite,sentrySiteData.toString()); - return sentrySite; - } - - @BeforeClass - public static void beforeTestSimpleSolrEndToEnd() throws Exception { - dfsCluster = HdfsTestUtil.setupClass(new File(Files.createTempDir(), - AbstractSolrSentryTestBase.class.getName() + "_" - + System.currentTimeMillis()).getAbsolutePath()); - File sentrySite = setupSentry(); - System.setProperty("solr.authorization.sentry.site", sentrySite.toURI().toURL().toString().substring("file:".length())); - System.setProperty("solr.hdfs.home", dfsCluster.getURI().toString() + "/solr"); - extraRequestFilters = new TreeMap<Class, String>(new Comparator<Class>() { - // There's only one class, make this as simple as possible - public int compare(Class o1, Class o2) { - return 0; - } - - public int hashCode() { - return 17; - } - - public boolean equals(Object obj) { - return true; - } - }); - extraRequestFilters.put(ModifiableUserAuthenticationFilter.class, "*"); - File solrXml = new File(RESOURCES_DIR, "solr-no-core.xml"); - miniSolrCloudCluster = new MiniSolrCloudCluster(NUM_SERVERS, null, solrXml, - null, extraRequestFilters); - } - - @AfterClass - public static void teardownClass() throws Exception { - HdfsTestUtil.teardownClass(dfsCluster); - System.clearProperty("solr.hdfs.home"); - System.clearProperty("solr.authorization.sentry.site"); - dfsCluster = null; - extraRequestFilters = null; - miniSolrCloudCluster.shutdown(); - } - - @Before - public void setupBeforeTest() throws Exception { - System.setProperty("solr.xml.persist", "true"); - // Disable the block cache because we can run out of memory - // on a MiniCluster. - System.setProperty("solr.hdfs.blockcache.enabled", "false"); - } - - @After - public void tearDown() throws Exception { - System.clearProperty("solr.hdfs.blockcache.enabled"); - System.clearProperty("solr.xml.persist"); - } - - /** - * Set the proper user in the Solr authentication filter - * @param solrUser - */ - protected void setAuthenticationUser(String solrUser) throws Exception { - ModifiableUserAuthenticationFilter.setUser(solrUser); - } - - /** - * Get the user defined in the Solr authentication filter - * @return - the username as String - * @throws Exception - */ - protected String getAuthenticatedUser() throws Exception { - return ModifiableUserAuthenticationFilter.getUser(); - } - - /** - * Function to return the user name based on the permissions provided. - * @param collectionName - Name of the solr collection. - * @param isQuery - Boolean that specifies query permission. - * @param isUpdate - Boolean that specifies update permission. - * @param isAll - Boolean that specifies all permission. - * @return - String which represents the Solr username. - */ - protected String getUsernameForPermissions(String collectionName, - boolean isQuery, - boolean isUpdate, - boolean isAll) { - StringBuilder finalStr = new StringBuilder(); - finalStr.append(collectionName); - finalStr.append("_"); - StringBuilder permissions = new StringBuilder(); - if (isQuery) { - permissions.append("q"); - } - - if (isUpdate) { - permissions.append("u"); - } - - if (isAll) { - permissions.append("a"); - } - - finalStr.append(permissions.toString()); - return finalStr.toString(); - } - - /** - * Method to validate Solr update passes - * @param solrUserName - User authenticated into Solr - * @param collectionName - Name of the collection to which the data has to be updated - * @param solrInputDoc - Instance of SolrInputDocument - * @throws Exception - */ - protected void verifyUpdatePass(String solrUserName, - String collectionName, - SolrInputDocument solrInputDoc) throws Exception { - String originalUser = getAuthenticatedUser(); - try { - SolrDocumentList orginalSolrDocs = getSolrDocs(collectionName, ALL_DOCS, true); - setAuthenticationUser(solrUserName); - CloudSolrServer cloudSolrServer = getCloudSolrServer(collectionName); - try { - cloudSolrServer.add(solrInputDoc); - cloudSolrServer.commit(); - } finally { - cloudSolrServer.shutdown(); - } - - orginalSolrDocs.add(ClientUtils.toSolrDocument(solrInputDoc)); - SolrDocumentList solrRespDocs = getSolrDocs(collectionName, ALL_DOCS, true); - // Validate Solr content to check whether the update command went through. - validateSolrDocCountAndContent(orginalSolrDocs, solrRespDocs); - } - finally { - setAuthenticationUser(originalUser); - } - } - - /** - * Method to validate Solr update fails - * @param solrUserName - User authenticated into Solr - * @param collectionName - Name of the collection to which the data has to be updated - * @param solrInputDoc - Instance of SolrInputDocument - * @throws Exception - */ - protected void verifyUpdateFail(String solrUserName, - String collectionName, - SolrInputDocument solrInputDoc) throws Exception { - String originalUser = getAuthenticatedUser(); - try { - SolrDocumentList orginalSolrDocs = getSolrDocs(collectionName, ALL_DOCS, true); - setAuthenticationUser(solrUserName); - CloudSolrServer cloudSolrServer = getCloudSolrServer(collectionName); - try { - cloudSolrServer.add(solrInputDoc); - cloudSolrServer.commit(); - fail("The specified user: " + solrUserName + " shouldn't get update access!"); - } catch (Exception exception) { - assertTrue("Expected " + SENTRY_ERROR_MSG + " in " + exception.toString(), - exception.toString().contains(SENTRY_ERROR_MSG)); - } finally { - cloudSolrServer.shutdown(); - } - - SolrDocumentList solrRespDocs = getSolrDocs(collectionName, ALL_DOCS, true); - // Validate Solr content to check whether the update command didn't go through. - validateSolrDocCountAndContent(orginalSolrDocs, solrRespDocs); - } finally { - setAuthenticationUser(originalUser); - } - } - - /** - * Method to validate Solr deletedocs passes - * (This function doesn't check if there is at least one Solr document present in Solr) - * @param solrUserName - User authenticated into Solr - * @param collectionName - Name of the collection to which the data has to be updated - * @param allowZeroDocs - Boolean for running this method only if there is atleast one Solr doc present. - * @throws MalformedURLException, SolrServerException, IOException - */ - protected void verifyDeletedocsPass(String solrUserName, - String collectionName, - boolean allowZeroDocs) throws Exception { - String originalUser = getAuthenticatedUser(); - try { - SolrDocumentList orginalSolrDocs = getSolrDocs(collectionName, ALL_DOCS, true); - if (allowZeroDocs == false) { - assertTrue("Solr should contain atleast one solr doc to run this test.", orginalSolrDocs.size() > 0); - } - - setAuthenticationUser(solrUserName); - CloudSolrServer cloudSolrServer = getCloudSolrServer(collectionName); - try { - cloudSolrServer.deleteByQuery(ALL_DOCS); - cloudSolrServer.commit(); - } finally { - cloudSolrServer.shutdown(); - } - - // Validate Solr doc count is zero - SolrDocumentList solrRespDocs = getSolrDocs(collectionName, ALL_DOCS, true); - validateSolrDocCountAndContent(new SolrDocumentList(), solrRespDocs); - } finally { - setAuthenticationUser(originalUser); - } - } - - /** - * Method to validate Solr deletedocs fails - * (This function doesn't check if there is at least one Solr document present in Solr) - * @param solrUserName - User authenticated into Solr - * @param collectionName - Name of the collection to which the data has to be updated - * @param allowZeroDocs - Boolean for running this method only if there is atleast one Solr doc present. - * @throws Exception - */ - protected void verifyDeletedocsFail(String solrUserName, - String collectionName, - boolean allowZeroDocs) throws Exception { - String originalUser = getAuthenticatedUser(); - try { - SolrDocumentList orginalSolrDocs = getSolrDocs(collectionName, ALL_DOCS, true); - if (allowZeroDocs == false) { - assertTrue("Solr should contain atleast one solr doc to run this test.", orginalSolrDocs.size() > 0); - } - - setAuthenticationUser(solrUserName); - CloudSolrServer cloudSolrServer = getCloudSolrServer(collectionName); - try { - cloudSolrServer.deleteByQuery(ALL_DOCS); - cloudSolrServer.commit(); - fail("The specified user: " + solrUserName + " shouldn't get deletedocs access!"); - } catch (Exception exception) { - assertTrue("Expected " + SENTRY_ERROR_MSG + " in " + exception.toString(), - exception.toString().contains(SENTRY_ERROR_MSG)); - } finally { - cloudSolrServer.shutdown(); - } - - // Validate Solr doc count and content is same as original set. - SolrDocumentList solrRespDocs = getSolrDocs(collectionName, ALL_DOCS, true); - validateSolrDocCountAndContent(orginalSolrDocs, solrRespDocs); - } finally { - setAuthenticationUser(originalUser); - } - } - - /** - * Method to validate Solr query passes - * @param solrUserName - User authenticated into Solr - * @param collectionName - Name of the collection to be queried - * @param solrQueryStr - Query string to be searched in Solr - * @throws Exception - */ - protected void verifyQueryPass(String solrUserName, - String collectionName, - String solrQueryStr) throws Exception { - String originalUser = getAuthenticatedUser(); - try { - SolrDocumentList orginalSolrDocs = getSolrDocs(collectionName, solrQueryStr, true); - setAuthenticationUser(solrUserName); - SolrDocumentList solrRespDocs = null; - solrRespDocs = getSolrDocs(collectionName, solrQueryStr, false); - - // Validate Solr content to check whether the query command went through. - validateSolrDocCountAndContent(orginalSolrDocs, solrRespDocs); - } finally { - setAuthenticationUser(originalUser); - } - } - - /** - * Method to validate Solr query fails - * @param solrUserName - User authenticated into Solr - * @param collectionName - Name of the collection to be queried - * @param solrQueryStr - Query string to be searched in Solr - * @throws Exception - */ - protected void verifyQueryFail(String solrUserName, - String collectionName, - String solrQueryStr) throws Exception { - String originalUser = getAuthenticatedUser(); - try { - setAuthenticationUser(solrUserName); - try { - getSolrDocs(collectionName, solrQueryStr, false); - fail("The specified user: " + solrUserName + " shouldn't get query access!"); - } catch (Exception exception) { - assertTrue("Expected " + SENTRY_ERROR_MSG + " in " + exception.toString(), - exception.toString().contains(SENTRY_ERROR_MSG)); - } - } finally { - setAuthenticationUser(originalUser); - } - } - - /** - * Method to validate collection Admin operation pass - * @param solrUserName - User authenticated into Solr - * @param adminOp - Admin operation to be performed - * @param collectionName - Name of the collection to be queried - * @throws Exception - */ - protected void verifyCollectionAdminOpPass(String solrUserName, - CollectionAction adminOp, - String collectionName) throws Exception { - verifyCollectionAdminOpPass(solrUserName, adminOp, collectionName, null); - } - - /** - * Method to validate collection Admin operation pass - * @param solrUserName - User authenticated into Solr - * @param adminOp - Admin operation to be performed - * @param collectionName - Name of the collection to be queried - * @param params - SolrParams to use - * @throws Exception - */ - protected void verifyCollectionAdminOpPass(String solrUserName, - CollectionAction adminOp, - String collectionName, - SolrParams params) throws Exception { - String originalUser = getAuthenticatedUser(); - try { - setAuthenticationUser(solrUserName); - QueryRequest request = populateCollectionAdminParams(adminOp, collectionName, params); - CloudSolrServer solrServer = createNewCloudSolrServer(); - try { - solrServer.request(request); - if (adminOp.compareTo(CollectionAction.CREATE) == 0) { - // Wait for collection creation to complete. - waitForRecoveriesToFinish(collectionName, solrServer, false); - } - } finally { - solrServer.shutdown(); - } - } finally { - setAuthenticationUser(originalUser); - } - } - - /** - * Method to validate collection Admin operation fail - * @param solrUserName - User authenticated into Solr - * @param adminOp - Admin operation to be performed - * @param collectionName - Name of the collection to be queried - * @throws Exception - */ - protected void verifyCollectionAdminOpFail(String solrUserName, - CollectionAction adminOp, - String collectionName) throws Exception { - verifyCollectionAdminOpFail(solrUserName, adminOp, collectionName, null); - } - - /** - * Method to validate collection Admin operation fail - * @param solrUserName - User authenticated into Solr - * @param adminOp - Admin operation to be performed - * @param collectionName - Name of the collection to be queried - * @param params - SolrParams to use - * @throws Exception - */ - protected void verifyCollectionAdminOpFail(String solrUserName, - CollectionAction adminOp, - String collectionName, - SolrParams params) throws Exception { - - String originalUser = getAuthenticatedUser(); - try { - setAuthenticationUser(solrUserName); - try { - QueryRequest request = populateCollectionAdminParams(adminOp, collectionName, params); - CloudSolrServer solrServer = createNewCloudSolrServer(); - try { - solrServer.request(request); - if (adminOp.compareTo(CollectionAction.CREATE) == 0) { - // Wait for collection creation to complete. - waitForRecoveriesToFinish(collectionName, solrServer, false); - } - } finally { - solrServer.shutdown(); - } - - fail("The specified user: " + solrUserName + " shouldn't get admin access for " + adminOp); - } catch (Exception exception) { - assertTrue("Expected " + SENTRY_ERROR_MSG + " in " + exception.toString(), - exception.toString().contains(SENTRY_ERROR_MSG)); - } - } finally { - setAuthenticationUser(originalUser); - } - } - - /** - * Method to populate the Solr params based on the collection admin being performed. - * @param adminOp - Collection admin operation - * @param collectionName - Name of the collection - * @return - instance of QueryRequest. - */ - public QueryRequest populateCollectionAdminParams(CollectionAction adminOp, - String collectionName) { - return populateCollectionAdminParams(adminOp, collectionName, null); - } - - /** - * Method to populate the Solr params based on the collection admin being performed. - * @param adminOp - Collection admin operation - * @param collectionName - Name of the collection - * @param params - SolrParams to use - * @return - instance of QueryRequest. - */ - public QueryRequest populateCollectionAdminParams(CollectionAction adminOp, - String collectionName, - SolrParams params) { - ModifiableSolrParams modParams = new ModifiableSolrParams(); - modParams.set(CoreAdminParams.ACTION, adminOp.name()); - switch (adminOp) { - case CREATE: - modParams.set("name", collectionName); - modParams.set("numShards", 2); - modParams.set("shards", "shard1,shard2"); - modParams.set("replicationFactor", 1); - break; - case DELETE: - modParams.set("name", collectionName); - break; - case RELOAD: - modParams.set("name", collectionName); - break; - case SPLITSHARD: - modParams.set("collection", collectionName); - modParams.set("shard", "shard1"); - break; - case DELETESHARD: - modParams.set("collection", collectionName); - modParams.set("shard", "shard1"); - break; - case CREATEALIAS: - modParams.set("name", collectionName); - modParams.set("collections", collectionName + "_underlying1" - + "," + collectionName + "_underlying2"); - break; - case DELETEALIAS: - modParams.set("name", collectionName); - break; - default: - throw new IllegalArgumentException("Admin operation: " + adminOp + " is not supported!"); - } - - if (params != null) { - Iterator<String> it = params.getParameterNamesIterator(); - while (it.hasNext()) { - String param = it.next(); - String [] value = params.getParams(param); - modParams.set(param, value); - } - } - QueryRequest request = new QueryRequest(modParams); - request.setPath("/admin/collections"); - return request; - } - - /** - * Function to validate the count and content of two SolrDocumentList's. - * @param solrOriginalDocs - Instance of initial set of solr docs before processing - * @param solrResponseDocs - Instance of response solr docs after processing - */ - protected void validateSolrDocCountAndContent(SolrDocumentList solrOriginalDocs, - SolrDocumentList solrResponseDocs) { - assertEquals("Expected number of Solr docs: " + solrOriginalDocs.size() + "; But found:" + solrResponseDocs.size(), - solrOriginalDocs.size(), solrResponseDocs.size()); - for (SolrDocument solrDoc : solrOriginalDocs) { - SolrInputDocument solrInputDoc = ClientUtils.toSolrInputDocument(solrDoc); - validateSolrDocContent(solrInputDoc, solrResponseDocs); - } - } - - /** - * Function to query the collection and fetch the Solr docs - * @param collectionName - Name of the collection - * @param solrQueryStr - Query string to be searched in Solr - * @param runAsAdmin - Boolean to specify whether to execute the Solr query as admin user - * @return - Instance of SolrDocumentList - * @throws Exception - */ - protected SolrDocumentList getSolrDocs(String collectionName, - String solrQueryStr, - boolean runAsAdmin) throws Exception { - String originalUser = getAuthenticatedUser(); - try { - if (runAsAdmin == true) { - // Authenticate as user "admin" - setAuthenticationUser(ADMIN_USER); - } - - CloudSolrServer cloudSolrServer = getCloudSolrServer(collectionName); - assertNotNull("Solr query shouldn't be null.", solrQueryStr); - SolrDocumentList solrDocs = null; - try { - SolrQuery query = new SolrQuery(solrQueryStr); - QueryResponse response = cloudSolrServer.query(query); - solrDocs = response.getResults(); - return solrDocs; - } finally { - cloudSolrServer.shutdown(); - } - } finally { - setAuthenticationUser(originalUser); - } - } - - /** - * Function to validate the content of Solr response with that of input document. - * @param solrInputDoc - Solr doc inserted into Solr - * @param solrRespDocs - List of Solr doc obtained as response - * (NOTE: This function ignores "_version_" field in validating Solr doc content) - */ - public void validateSolrDocContent(SolrInputDocument solrInputDoc, - SolrDocumentList solrRespDocs) { - for (SolrDocument solrRespDoc : solrRespDocs) { - String expFieldValue = (String) solrInputDoc.getFieldValue("id"); - String resFieldValue = (String) solrRespDoc.getFieldValue("id"); - if (expFieldValue.equals(resFieldValue)) { - int expectedRespFieldCount = solrRespDoc.size(); - if (solrRespDoc.containsKey("_version_")) { - expectedRespFieldCount = expectedRespFieldCount - 1; - } - int expectedOrigFieldCount = solrInputDoc.size(); - if (solrInputDoc.containsKey("_version_")) { - expectedOrigFieldCount = expectedOrigFieldCount - 1; - } - assertEquals("Expected " + expectedOrigFieldCount + " fields. But, found " - + expectedRespFieldCount + " fields", expectedOrigFieldCount , expectedRespFieldCount); - for (String field : solrInputDoc.getFieldNames()) { - if (field.equals("_version_") == true) { - continue; - } - - expFieldValue = (String) solrInputDoc.getFieldValue(field); - resFieldValue = (String) solrRespDoc.getFieldValue(field); - assertEquals("Expected value for field: " + field + " is " + expFieldValue - + "; But, found " + resFieldValue, expFieldValue, resFieldValue); - } - - return; - } - } - - fail("Solr doc not found in Solr collection"); - } - - /** - * Function to return the instance of CloudSolrServer for the collectionName specified - * @param collectionName - Name of the collection - * @return instance of CloudSolrServer - * @throws MalformedURLException - */ - protected CloudSolrServer getCloudSolrServer(String collectionName) throws MalformedURLException { - CloudSolrServer cloudSolrServer = new CloudSolrServer(miniSolrCloudCluster.getZkServer().getZkAddress(), - RANDOM.nextBoolean()); - cloudSolrServer.setDefaultCollection(collectionName); - cloudSolrServer.connect(); - return cloudSolrServer; - } - - /** - * Function to create a solr collection with the name passed as parameter - * (Runs commands as ADMIN user) - * @param collectionName - Name of the collection - * @throws Exception - */ - protected void setupCollection(String collectionName) throws Exception { - verifyCollectionAdminOpPass(ADMIN_USER, - CollectionAction.CREATE, - collectionName); - } - - /** - * Function to delete a solr collection with the name passed as parameter - * (Runs commands as ADMIN user) - * @param collectionName - Name of the collection - * This function will simply ignore the errors raised in deleting the collections. - * e.g: As part of the clean up job, the tests can issue a DELETE command on the collection which doesn't exist. - */ - protected void deleteCollection(String collectionName) { - try { - verifyCollectionAdminOpPass(ADMIN_USER, - CollectionAction.DELETE, - collectionName); - } catch (Exception e) { - LOG.warn("Ignoring errors raised while deleting the collection : " + e.toString()); - } - } - - /** - * Function to clean Solr collections - * @param collectionName - Name of the collection - * @throws Exception - */ - protected void cleanSolrCollection(String collectionName) - throws Exception { - verifyDeletedocsPass(ADMIN_USER, collectionName, true); - } - - /** - * Function to create a test Solrdoc with a random number as the ID - * @throws Exception - */ - protected SolrInputDocument createSolrTestDoc() throws Exception { - SolrInputDocument solrInputDoc = new SolrInputDocument(); - String solrDocId = String.valueOf(RANDOM.nextInt()); - solrInputDoc.addField("id", solrDocId); - solrInputDoc.addField("name", "testdoc" + solrDocId); - return solrInputDoc; - } - - /** - * Load Solr collection with the SolrDocument passed. - * @param collectionName - Name of the Solr collection - * @param solrInputDoc - Solr document to be uploaded - * (If solrInputDoc is null, then a test Solr doc will be uploaded) - * @throws Exception - */ - protected void uploadSolrDoc(String collectionName, - SolrInputDocument solrInputDoc) throws Exception { - if (solrInputDoc == null) { - solrInputDoc = createSolrTestDoc(); - } - - verifyUpdatePass(ADMIN_USER, collectionName, solrInputDoc); - } - - private ZkController getZkController() { - SolrDispatchFilter dispatchFilter = - (SolrDispatchFilter) miniSolrCloudCluster.getJettySolrRunners().get(0).getDispatchFilter().getFilter(); - return dispatchFilter.getCores().getZkController(); - } - - protected void uploadConfigDirToZk(String collectionConfigDir) throws Exception { - uploadConfigDirToZk(collectionConfigDir, CONF_DIR_IN_ZK); - } - - protected void uploadConfigDirToZk(String collectionConfigDir, String confDirInZk) throws Exception { - ZkController zkController = getZkController(); - zkController.uploadConfigDir(new File(collectionConfigDir), confDirInZk); - } - - protected void uploadConfigFileToZk(String file, String nameInZk) throws Exception { - uploadConfigFileToZk(file, nameInZk, CONF_DIR_IN_ZK); - } - - protected void uploadConfigFileToZk(String file, String nameInZk, String confDirInZk) throws Exception { - ZkController zkController = getZkController(); - zkController.getZkClient().makePath(ZkController.CONFIGS_ZKNODE + "/" - + confDirInZk + "/" + nameInZk, new File(file), false, true); - } - - protected CloudSolrServer createNewCloudSolrServer() throws Exception { - CloudSolrServer css = new CloudSolrServer(miniSolrCloudCluster.getZkServer().getZkAddress()); - css.connect(); - return css; - } - - /** - * Make a raw http request to specific cluster node. Node is of the format - * host:port/context, i.e. "localhost:8983/solr" - */ - protected String makeHttpRequest(CloudSolrServer server, String node, String httpMethod, - String path, byte [] content, String contentType) throws Exception { - HttpClient httpClient = server.getLbServer().getHttpClient(); - URI uri = new URI("http://"; + node + path); - HttpRequestBase method = null; - if ("GET".equals(httpMethod)) { - method = new HttpGet(uri); - } else if ("HEAD".equals(httpMethod)) { - method = new HttpHead(uri); - } else if ("POST".equals(httpMethod)) { - method = new HttpPost(uri); - } else if ("PUT".equals(httpMethod)) { - method = new HttpPut(uri); - } else { - throw new IOException("Unsupported method: " + method); - } - - if (method instanceof HttpEntityEnclosingRequestBase) { - HttpEntityEnclosingRequestBase entityEnclosing = - (HttpEntityEnclosingRequestBase)method; - ByteArrayEntity entityRequest = new ByteArrayEntity(content); - entityRequest.setContentType(contentType); - entityEnclosing.setEntity(entityRequest); - } - - HttpEntity httpEntity = null; - boolean success = false; - String retValue = ""; - try { - final HttpResponse response = httpClient.execute(method); - httpEntity = response.getEntity(); - - if (httpEntity != null) { - InputStream is = httpEntity.getContent(); - ByteArrayOutputStream os = new ByteArrayOutputStream(); - try { - IOUtils.copyLarge(is, os); - os.flush(); - } finally { - IOUtils.closeQuietly(os); - IOUtils.closeQuietly(is); - } - retValue = os.toString(); - } - success = true; - } finally { - if (!success) { - EntityUtils.consumeQuietly(httpEntity); - method.abort(); - } - } - return retValue; - } - - /** - * Make a raw http request (not specifying cluster node) - */ - protected String makeHttpRequest(CloudSolrServer server, String httpMethod, - String path, byte [] content, String contentType) throws Exception { - Set<String> liveNodes = - server.getZkStateReader().getClusterState().getLiveNodes(); - assertTrue("Expected at least one live node", !liveNodes.isEmpty()); - String firstServer = liveNodes.toArray(new String[0])[0].replace("_solr", "/solr"); - return makeHttpRequest(server, firstServer, httpMethod, path, content, contentType); - } - - protected static void waitForRecoveriesToFinish(String collection, - CloudSolrServer solrServer, - boolean verbose) throws Exception { - waitForRecoveriesToFinish(collection, solrServer, verbose, true, 60); - } - - protected static void waitForRecoveriesToFinish(String collection, - CloudSolrServer solrServer, - boolean verbose, - boolean failOnTimeout, - int timeoutSeconds) throws Exception { - LOG.info("Entering solr wait with timeout " + timeoutSeconds); - ZkStateReader zkStateReader = solrServer.getZkStateReader(); - try { - boolean cont = true; - int cnt = 0; - - while (cont) { - if (verbose) { - LOG.debug("-"); - } - boolean sawLiveRecovering = false; - zkStateReader.updateClusterState(true); - ClusterState clusterState = zkStateReader.getClusterState(); - Map<String, Slice> slices = clusterState.getSlicesMap(collection); - assertNotNull("Could not find collection:" + collection, slices); - for (Map.Entry<String, Slice> entry : slices.entrySet()) { - Map<String, Replica> shards = entry.getValue().getReplicasMap(); - for (Map.Entry<String, Replica> shard : shards.entrySet()) { - if (verbose) { - LOG.debug("rstate:" - + shard.getValue().getStr(ZkStateReader.STATE_PROP) + " live:" - + clusterState.liveNodesContain(shard.getValue().getNodeName())); - } - String state = shard.getValue().getStr(ZkStateReader.STATE_PROP); - if ((state.equals(ZkStateReader.RECOVERING) - || state.equals(ZkStateReader.SYNC) || state - .equals(ZkStateReader.DOWN)) - && clusterState.liveNodesContain(shard.getValue().getStr( - ZkStateReader.NODE_NAME_PROP))) { - sawLiveRecovering = true; - } - } - } - if (!sawLiveRecovering || cnt == timeoutSeconds) { - if (!sawLiveRecovering) { - if (verbose) { - LOG.debug("no one is recovering"); - } - } else { - if (verbose) { - LOG.debug("Gave up waiting for recovery to finish.."); - } - if (failOnTimeout) { - fail("There are still nodes recovering - waited for " - + timeoutSeconds + " seconds"); - // won't get here - return; - } - } - cont = false; - } else { - Thread.sleep(1000); - } - cnt++; - } - } finally { - LOG.info("Exiting solr wait"); - } - } -}
http://git-wip-us.apache.org/repos/asf/sentry/blob/e62fa28d/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestCase.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestCase.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestCase.java new file mode 100644 index 0000000..e6e7a70 --- /dev/null +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestCase.java @@ -0,0 +1,600 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.tests.e2e.solr; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.Thread.UncaughtExceptionHandler; +import java.net.MalformedURLException; +import java.net.URI; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.Locale; + +import javax.servlet.http.HttpServletResponse; + +import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.util.EntityUtils; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; + +import org.apache.sentry.binding.solr.authz.SentrySolrPluginImpl; +import org.apache.sentry.core.common.exception.SentryUserException; +import org.apache.sentry.core.model.solr.SolrConstants; +import org.apache.sentry.core.model.solr.SolrModelAuthorizable; +import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClient; +import org.apache.sentry.provider.db.generic.service.thrift.TAuthorizable; +import org.apache.sentry.provider.db.generic.service.thrift.TSentryGrantOption; +import org.apache.sentry.provider.db.generic.service.thrift.TSentryPrivilege; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.util.Utils; +import org.apache.solr.SolrTestCaseJ4; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.restlet.representation.Representation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +@SolrTestCaseJ4.SuppressSSL +public abstract class AbstractSolrSentryTestCase extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(AbstractSolrSentryTestCase.class); + protected static final int NUM_SERVERS = 3; + protected static final String COMPONENT_SOLR = "solr"; + protected static final String SERVICE_NAME = "service1"; + protected static final String ADMIN_ROLE = "admin_role"; + private static final String SENTRY_SITE_LOC_SYSPROP = "solr."+SentrySolrPluginImpl.SNTRY_SITE_LOCATION_PROPERTY; + protected static final String ALL_DOCS = "*:*"; + + protected static TestSentryServer sentrySvc; + protected static SentryGenericServiceClient sentryClient; + protected static UncaughtExceptionHandler orgExceptionHandler; + + @BeforeClass + public static void setupClass() throws Exception { + skipBrokenLocales(); + + Path testDataPath = createTempDir("solr-integration-db-"); + + try { + sentrySvc = new TestSentryServer(testDataPath, getUserGroupMappings()); + sentrySvc.startSentryService(); + sentryClient = sentrySvc.connectToSentryService(); + log.info("Successfully started Sentry service"); + } catch (Exception ex) { + log.error ("Unexpected exception while starting Sentry service", ex); + throw ex; + } + + for (int i = 0; i < 4; i++) { + sentryClient.createRole(TestSentryServer.ADMIN_USER, "role"+i, COMPONENT_SOLR); + sentryClient.grantRoleToGroups(TestSentryServer.ADMIN_USER, "role"+i, COMPONENT_SOLR, Collections.singleton("group"+i)); + } + + log.info("Successfully created roles in Sentry service"); + + sentryClient.createRole(TestSentryServer.ADMIN_USER, ADMIN_ROLE, COMPONENT_SOLR); + sentryClient.grantRoleToGroups(TestSentryServer.ADMIN_USER, ADMIN_ROLE, COMPONENT_SOLR, + Collections.singleton(TestSentryServer.ADMIN_GROUP)); + grantAdminPrivileges(TestSentryServer.ADMIN_USER, ADMIN_ROLE, SolrConstants.ALL, SolrConstants.ALL); + + log.info("Successfully granted admin privileges to " + ADMIN_ROLE); + + + System.setProperty(SENTRY_SITE_LOC_SYSPROP, sentrySvc.getSentrySitePath().toString()); + + // set the solr for the loginUser and belongs to solr group + // Note - Solr/Sentry unit tests don't use Hadoop authentication framework. Hence the + // UserGroupInformation is not available when the request is being processed by the Solr server. + // The document level security search component requires this UserGroupInformation while querying + // the roles associated with the user. Please refer to implementation of + // SentryGenericProviderBackend#getRoles(...) method. Hence this is a workaround to satisfy this requirement. + UserGroupInformation.setLoginUser(UserGroupInformation.createUserForTesting("solr", new String[]{"solr"})); + + try { + configureCluster(NUM_SERVERS) + .withSecurityJson(TEST_PATH().resolve("security").resolve("security.json")) + .addConfig("cloud-minimal", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .addConfig("cloud-managed", TEST_PATH().resolve("configsets").resolve("cloud-managed").resolve("conf")) + .addConfig("cloud-minimal_doc_level_security", TEST_PATH().resolve("configsets") + .resolve("cloud-minimal_doc_level_security").resolve("conf")) + .configure(); + log.info("Successfully started Solr service"); + + } catch (Exception ex) { + log.error ("Unexpected exception while starting SolrCloud", ex); + throw ex; + } + + log.info("Successfully setup Solr with Sentry service"); + } + + /** + * A sample failed run was with following parameters + * -Dtests.seed=C92C593AE70E7466 -Dtests.locale=und -Dtests.timezone=Europe/Sarajevo -Dtests.asserts=true -Dtests.file.encoding=UTF-8 + */ + private static void skipBrokenLocales() { + assumeFalse("This test fails on UNIX whenever Locale has a language, country, or variant that does not satisfy" + + " the IETF BCP 47 language tag syntax requirements", + "und".equals(Locale.getDefault().toLanguageTag())); + assumeFalse("This test fails on UNIX with error - Supplied locale description 'sr__#Latn' is invalid, expecting ln[_CO[_variant]]" + + "ln=lower-case two-letter ISO-639 language code, CO=upper-case two-letter ISO-3166 country code", + "sr-Latn".equals(Locale.getDefault().toLanguageTag())); + } + + @AfterClass + public static void cleanUpResources() throws Exception { + if (sentryClient != null) { + sentryClient.close(); + sentryClient = null; + } + if (sentrySvc != null) { + sentrySvc.close(); + sentrySvc = null; + } + System.clearProperty(SENTRY_SITE_LOC_SYSPROP); + } + + public static void grantAdminPrivileges(String requestor, String roleName, String entityName, String action) + throws SentryUserException { + List<TAuthorizable> auths = new ArrayList<>(1); + auths.add(new TAuthorizable(SolrModelAuthorizable.AuthorizableType.Admin.name(), entityName)); + + TSentryPrivilege privilege = new TSentryPrivilege(COMPONENT_SOLR, SERVICE_NAME, auths, action); + privilege.setGrantOption(TSentryGrantOption.TRUE); + sentryClient.grantPrivilege(requestor, roleName, COMPONENT_SOLR, privilege); + } + + public static void revokeAdminPrivileges(String requestor, String roleName, String entityName, String action) + throws SentryUserException { + List<TAuthorizable> auths = new ArrayList<>(1); + auths.add(new TAuthorizable(SolrModelAuthorizable.AuthorizableType.Admin.name(), entityName)); + + TSentryPrivilege privilege = new TSentryPrivilege(COMPONENT_SOLR, SERVICE_NAME, auths, action); + privilege.setGrantOption(TSentryGrantOption.TRUE); + sentryClient.revokePrivilege(requestor, roleName, COMPONENT_SOLR, privilege); + } + + public static void grantCollectionPrivileges(String requestor, String roleName, + String collectionName, String action) throws SentryUserException { + List<TAuthorizable> auths = new ArrayList<>(1); + auths.add(new TAuthorizable(SolrModelAuthorizable.AuthorizableType.Collection.name(), collectionName)); + + TSentryPrivilege privilege = new TSentryPrivilege(COMPONENT_SOLR, SERVICE_NAME, auths, action); + privilege.setGrantOption(TSentryGrantOption.TRUE); + sentryClient.grantPrivilege(requestor, roleName, COMPONENT_SOLR, privilege); + } + + public static void revokeCollectionPrivileges(String requestor, String roleName, + String collectionName, String action) throws SentryUserException { + List<TAuthorizable> auths = new ArrayList<>(1); + auths.add(new TAuthorizable(SolrModelAuthorizable.AuthorizableType.Collection.name(), collectionName)); + + TSentryPrivilege privilege = new TSentryPrivilege(COMPONENT_SOLR, SERVICE_NAME, auths, action); + privilege.setGrantOption(TSentryGrantOption.TRUE); + sentryClient.revokePrivilege(requestor, roleName, COMPONENT_SOLR, privilege); + } + + public static void grantConfigPrivileges(String requestor, String roleName, String configName, + String action) throws SentryUserException { + List<TAuthorizable> auths = new ArrayList<>(1); + auths.add(new TAuthorizable(SolrModelAuthorizable.AuthorizableType.Config.name(), configName)); + + TSentryPrivilege privilege = new TSentryPrivilege(COMPONENT_SOLR, SERVICE_NAME, auths, action); + privilege.setGrantOption(TSentryGrantOption.TRUE); + sentryClient.grantPrivilege(requestor, roleName, COMPONENT_SOLR, privilege); + } + + public static void revokeConfigPrivileges(String requestor, String roleName, String configName, + String action) throws SentryUserException { + List<TAuthorizable> auths = new ArrayList<>(1); + auths.add(new TAuthorizable(SolrModelAuthorizable.AuthorizableType.Config.name(), configName)); + + TSentryPrivilege privilege = new TSentryPrivilege(COMPONENT_SOLR, SERVICE_NAME, auths, action); + privilege.setGrantOption(TSentryGrantOption.TRUE); + sentryClient.revokePrivilege(requestor, roleName, COMPONENT_SOLR, privilege); + } + + public static void grantSchemaPrivileges(String requestor, String roleName, String schemaName, + String action) throws SentryUserException { + List<TAuthorizable> auths = new ArrayList<>(1); + auths.add(new TAuthorizable(SolrModelAuthorizable.AuthorizableType.Schema.name(), schemaName)); + + TSentryPrivilege privilege = new TSentryPrivilege(COMPONENT_SOLR, SERVICE_NAME, auths, action); + privilege.setGrantOption(TSentryGrantOption.TRUE); + sentryClient.grantPrivilege(requestor, roleName, COMPONENT_SOLR, privilege); + } + + public static void revokeSchemaPrivileges(String requestor, String roleName, String schemaName, + String action) throws SentryUserException { + List<TAuthorizable> auths = new ArrayList<>(1); + auths.add(new TAuthorizable(SolrModelAuthorizable.AuthorizableType.Schema.name(), schemaName)); + + TSentryPrivilege privilege = new TSentryPrivilege(COMPONENT_SOLR, SERVICE_NAME, auths, action); + privilege.setGrantOption(TSentryGrantOption.TRUE); + sentryClient.revokePrivilege(requestor, roleName, COMPONENT_SOLR, privilege); + } + + protected static Map<String, Set<String>> getUserGroupMappings() { + Map<String, Set<String>> result = new HashMap<>(); + result.put("user0", Collections.singleton("group0")); + result.put("user1", Collections.singleton("group1")); + result.put("user2", Collections.singleton("group2")); + result.put("user3", Collections.singleton("group3")); + result.put(TestSentryServer.ADMIN_USER, Collections.singleton(TestSentryServer.ADMIN_GROUP)); + result.put("solr", Collections.singleton("solr")); + result.put("junit", Collections.singleton("junit")); + result.put("doclevel", Collections.singleton("doclevel")); + return Collections.unmodifiableMap(result); + } + + /** + * Get the user defined in the Solr authentication plugin + * + * @return - the username as String + * @throws Exception in case of errors + */ + protected String getAuthenticatedUser() { + return DummyAuthPluginImpl.getUserName(); + } + + /** + * Set the proper user in the Solr authentication plugin + * @param solrUser + */ + protected void setAuthenticationUser(String solrUser) { + DummyAuthPluginImpl.setUserName(solrUser); + } + + protected void createCollection (String userName, String collectionName, + String configName, int numShards, int numReplicas) throws SolrServerException, IOException { + String tmp = getAuthenticatedUser(); + try { + setAuthenticationUser(userName); + // Create collection. + CollectionAdminRequest.Create createCmd = + CollectionAdminRequest.createCollection(collectionName, configName, numShards, numReplicas); + assertEquals(0, createCmd.process(cluster.getSolrClient()).getStatus()); + } finally { + setAuthenticationUser(tmp); + } + } + + /** + * Function to clean Solr collections + * @param userName Name of the user performing this operation + * @param collectionName - Name of the collection + * @throws Exception In case of error + */ + protected void cleanSolrCollection(String userName, String collectionName) + throws Exception { + verifyDeletedocsPass(userName, collectionName, true); + } + + /** + * Method to validate Solr deletedocs passes + * (This function doesn't check if there is at least one Solr document present in Solr) + * @param solrUserName - User authenticated into Solr + * @param collectionName - Name of the collection to which the data has to be updated + * @param allowZeroDocs - Boolean for running this method only if there is atleast one Solr doc present. + * @throws MalformedURLException, SolrServerException, IOException + */ + protected void verifyDeletedocsPass(String solrUserName, + String collectionName, + boolean allowZeroDocs) throws Exception { + String originalUser = getAuthenticatedUser(); + try { + SolrDocumentList orginalSolrDocs = getSolrDocs(solrUserName, collectionName, ALL_DOCS); + if (!allowZeroDocs) { + assertTrue("Solr should contain atleast one solr doc to run this test.", orginalSolrDocs.size() > 0); + } + + setAuthenticationUser(solrUserName); + cluster.getSolrClient().deleteByQuery(collectionName, ALL_DOCS); + cluster.getSolrClient().commit(collectionName); + + // Validate Solr doc count is zero + SolrDocumentList solrRespDocs = getSolrDocs(solrUserName, collectionName, ALL_DOCS); + validateSolrDocCountAndContent(new SolrDocumentList(), solrRespDocs); + } finally { + setAuthenticationUser(originalUser); + } + } + + /** + * Function to query the collection and fetch the Solr docs + * @param userName The name of the user performing this operation. + * @param collectionName - Name of the collection + * @param solrQueryStr - Query string to be searched in Solr + * @return an instance of SolrDocumentList + * @throws Exception + */ + protected SolrDocumentList getSolrDocs(String userName, + String collectionName, + String solrQueryStr) throws Exception { + String originalUser = getAuthenticatedUser(); + try { + setAuthenticationUser(userName); + + assertNotNull("Solr query shouldn't be null.", solrQueryStr); + QueryResponse response = cluster.getSolrClient().query(collectionName, new SolrQuery(solrQueryStr)); + assertEquals(0, response.getStatus()); + return response.getResults(); + } finally { + setAuthenticationUser(originalUser); + } + } + + /** + * Function to validate the count and content of two SolrDocumentList's. + * @param solrOriginalDocs - Instance of initial set of solr docs before processing + * @param solrResponseDocs - Instance of response solr docs after processing + */ + protected void validateSolrDocCountAndContent(SolrDocumentList solrOriginalDocs, + SolrDocumentList solrResponseDocs) { + assertEquals("Expected number of Solr docs: " + solrOriginalDocs.size() + "; But found:" + solrResponseDocs.size(), + solrOriginalDocs.size(), solrResponseDocs.size()); + for (SolrDocument solrDoc : solrOriginalDocs) { + validateSolrDocContent(solrDoc, solrResponseDocs); + } + } + + /** + * Function to validate the content of Solr response with that of input document. + * @param solrInputDoc - Solr doc inserted into Solr + * @param solrRespDocs - List of Solr doc obtained as response + * (NOTE: This function ignores "_version_" field in validating Solr doc content) + */ + public void validateSolrDocContent(SolrDocument solrInputDoc, + SolrDocumentList solrRespDocs) { + for (SolrDocument solrRespDoc : solrRespDocs) { + String expFieldValue = (String) solrInputDoc.getFieldValue("id"); + String resFieldValue = (String) solrRespDoc.getFieldValue("id"); + if (expFieldValue.equals(resFieldValue)) { + int expectedRespFieldCount = solrRespDoc.size(); + if (solrRespDoc.containsKey("_version_")) { + expectedRespFieldCount = expectedRespFieldCount - 1; + } + int expectedOrigFieldCount = solrInputDoc.size(); + if (solrInputDoc.containsKey("_version_")) { + expectedOrigFieldCount = expectedOrigFieldCount - 1; + } + assertEquals("Expected " + expectedOrigFieldCount + " fields. But, found " + + expectedRespFieldCount + " fields", expectedOrigFieldCount , expectedRespFieldCount); + for (String field : solrInputDoc.getFieldNames()) { + if (field.equals("_version_") == true) { + continue; + } + + expFieldValue = (String) solrInputDoc.getFieldValue(field); + resFieldValue = (String) solrRespDoc.getFieldValue(field); + assertEquals("Expected value for field: " + field + " is " + expFieldValue + + "; But, found " + resFieldValue, expFieldValue, resFieldValue); + } + + return; + } + } + + fail("Solr doc not found in Solr collection"); + } + + /** + * Function to create a test Solrdoc with a random number as the ID + * @throws Exception in case of error + */ + protected SolrInputDocument createSolrTestDoc() throws Exception { + String solrDocId = String.valueOf(random().nextInt()); + + SolrInputDocument doc = new SolrInputDocument(); + doc.setField("id", String.valueOf(random().nextInt())); + doc.setField("name", "testdoc" + solrDocId); + return doc; + } + + /** + * Method to validate Solr update passes + * @param solrUserName - User authenticated into Solr + * @param collectionName - Name of the collection to which the data has to be updated + * @param solrInputDoc - Instance of SolrInputDocument + * @throws Exception + */ + protected void verifyUpdatePass(String solrUserName, + String collectionName, + SolrInputDocument solrInputDoc) throws Exception { + String originalUser = getAuthenticatedUser(); + try { + SolrDocumentList orginalSolrDocs = getSolrDocs(solrUserName, collectionName, ALL_DOCS); + setAuthenticationUser(solrUserName); + + cluster.getSolrClient().add(collectionName, solrInputDoc); + cluster.getSolrClient().commit(collectionName); + + orginalSolrDocs.add(toSolrDocument(solrInputDoc)); + SolrDocumentList solrRespDocs = getSolrDocs(solrUserName, collectionName, ALL_DOCS); + // Validate Solr content to check whether the update command went through. + validateSolrDocCountAndContent(orginalSolrDocs, solrRespDocs); + } + finally { + setAuthenticationUser(originalUser); + } + } + + /** + * Method to validate Solr query fails + * @param solrUserName - User authenticated into Solr + * @param collectionName - Name of the collection to be queried + * @param solrQueryStr - Query string to be searched in Solr + * @throws Exception + */ + protected void verifyQueryFail(String solrUserName, + String collectionName, + String solrQueryStr) throws Exception { + try { + getSolrDocs(solrUserName, collectionName, solrQueryStr); + fail("The specified user: " + solrUserName + " shouldn't get query access!"); + } catch (SolrServerException exception) { + assertTrue(exception.getCause() instanceof RemoteSolrException); + assertEquals(HttpServletResponse.SC_FORBIDDEN, ((RemoteSolrException)exception.getCause()).code()); + } + } + + + /** + * Load Solr collection with the SolrDocument passed. + * @param collectionName - Name of the Solr collection + * @param solrInputDoc - Solr document to be uploaded + * (If solrInputDoc is null, then a test Solr doc will be uploaded) + * @throws Exception + */ + protected void uploadSolrDoc(String userName, + String collectionName, + SolrInputDocument solrInputDoc) throws Exception { + Objects.requireNonNull(solrInputDoc); + verifyUpdatePass(userName, collectionName, solrInputDoc); + } + + + private static SolrDocument toSolrDocument(SolrInputDocument doc) { + SolrDocument result = new SolrDocument(); + result.setField("id", doc.getFieldValue("id")); + result.setField("name", doc.getFieldValue("name")); + return result; + } + + @SuppressWarnings("unchecked") + protected <T> T deserialize (Representation r) throws IOException { + ByteArrayOutputStream str = new ByteArrayOutputStream(); + r.write(str); + return (T)Utils.fromJSON(str.toByteArray()); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + protected <T> T readNestedElement(Map object, String... fields) { + Map t = object; + int i = 0; + + while (i < fields.length - 1) { + String field = fields[i]; + t = (Map)Objects.requireNonNull(t.get(field)); + i++; + } + + return (T)Objects.requireNonNull(t.get(fields[fields.length - 1])); + } + + /** + * Make a raw http request to specific cluster node. Node is of the format + * host:port/context, i.e. "localhost:8983/solr" + */ + protected String makeHttpRequest(CloudSolrClient client, String node, String httpMethod, + String path, byte [] content, String contentType, int expectedStatusCode) throws Exception { + HttpClient httpClient = client.getLbClient().getHttpClient(); + URI uri = new URI("http://"; + node + path); + HttpRequestBase method = null; + if ("GET".equals(httpMethod)) { + method = new HttpGet(uri); + } else if ("HEAD".equals(httpMethod)) { + method = new HttpHead(uri); + } else if ("POST".equals(httpMethod)) { + method = new HttpPost(uri); + } else if ("PUT".equals(httpMethod)) { + method = new HttpPut(uri); + } else { + throw new IOException("Unsupported method: " + method); + } + + if (method instanceof HttpEntityEnclosingRequestBase) { + HttpEntityEnclosingRequestBase entityEnclosing = + (HttpEntityEnclosingRequestBase)method; + ByteArrayEntity entityRequest = new ByteArrayEntity(content); + entityRequest.setContentType(contentType); + entityEnclosing.setEntity(entityRequest); + } + + HttpEntity httpEntity = null; + boolean success = false; + String retValue = ""; + try { + final HttpResponse response = httpClient.execute(method); + httpEntity = response.getEntity(); + + assertEquals (expectedStatusCode, response.getStatusLine().getStatusCode()); + + if (httpEntity != null) { + InputStream is = httpEntity.getContent(); + ByteArrayOutputStream os = new ByteArrayOutputStream(); + try { + IOUtils.copyLarge(is, os); + os.flush(); + } finally { + IOUtils.closeQuietly(os); + IOUtils.closeQuietly(is); + } + retValue = os.toString(); + } + success = true; + } finally { + if (!success) { + EntityUtils.consumeQuietly(httpEntity); + method.abort(); + } + } + return retValue; + } + + /** + * Make a raw http request (not specifying cluster node) + */ + protected String makeHttpRequest(CloudSolrClient client, String httpMethod, + String path, byte [] content, String contentType, int expectedStatusCode) throws Exception { + Set<String> liveNodes = + client.getZkStateReader().getClusterState().getLiveNodes(); + assertTrue("Expected at least one live node", !liveNodes.isEmpty()); + String firstServer = liveNodes.toArray(new String[0])[0].replace("_solr", "/solr"); + return makeHttpRequest(client, firstServer, httpMethod, path, content, contentType, expectedStatusCode); + } + + +} http://git-wip-us.apache.org/repos/asf/sentry/blob/e62fa28d/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/DocLevelGenerator.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/DocLevelGenerator.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/DocLevelGenerator.java index e50e3f8..40cc153 100644 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/DocLevelGenerator.java +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/DocLevelGenerator.java @@ -16,28 +16,30 @@ */ package org.apache.sentry.tests.e2e.solr; +import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.client.solrj.impl.CloudSolrServer; import java.util.ArrayList; public class DocLevelGenerator { + private String collection; private String authField; - public DocLevelGenerator(String authField) { + public DocLevelGenerator(String collection, String authField) { + this.collection = collection; this.authField = authField; } /** * Generates docs according to the following parameters: * - * @param server SolrServer to use + * @param client Solr client to use * @param numDocs number of documents to generate * @param evenDocsToken every even number doc gets this token added to the authField * @param oddDocsToken every odd number doc gets this token added to the authField * @param extraAuthFieldsCount generates this number of bogus entries in the authField */ - public void generateDocs(CloudSolrServer server, int numDocs, String evenDocsToken, String oddDocsToken, int extraAuthFieldsCount) throws Exception { + public void generateDocs(CloudSolrClient client, int numDocs, String evenDocsToken, String oddDocsToken, int extraAuthFieldsCount) throws Exception { // create documents ArrayList<SolrInputDocument> docs = new ArrayList<SolrInputDocument>(); @@ -59,12 +61,12 @@ public class DocLevelGenerator { } // add a token to all docs so we can check that we can get all // documents returned - doc.addField(authField, "docLevel_role"); + doc.addField(authField, "doclevel_role"); docs.add(doc); } - server.add(docs); - server.commit(true, true); + client.add(collection, docs); + client.commit(collection, true, true); } } http://git-wip-us.apache.org/repos/asf/sentry/blob/e62fa28d/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/DummyAuthPluginImpl.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/DummyAuthPluginImpl.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/DummyAuthPluginImpl.java new file mode 100644 index 0000000..33cc29f --- /dev/null +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/DummyAuthPluginImpl.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.tests.e2e.solr; + +import java.io.IOException; +import java.security.Principal; +import java.util.Map; + +import javax.servlet.FilterChain; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; + +import org.apache.http.auth.BasicUserPrincipal; +import org.apache.solr.security.AuthenticationPlugin; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DummyAuthPluginImpl extends AuthenticationPlugin { + private static final Logger LOG = LoggerFactory.getLogger(DummyAuthPluginImpl.class); + + private static String userName = TestSentryServer.ADMIN_USER; + + @Override + public void init(Map<String, Object> arg0) { + } + + @Override + public void close() throws IOException { + } + + @Override + public boolean doAuthenticate(ServletRequest arg0, ServletResponse arg1, FilterChain arg2) + throws Exception { + HttpServletRequestWrapper wrapper = new HttpServletRequestWrapper((HttpServletRequest)arg0) { + @Override + public Principal getUserPrincipal() { + return new BasicUserPrincipal(getUserName()); + } + }; + arg2.doFilter(wrapper, arg1); + return true; + } + + public static synchronized void setUserName(String userName) { + LOG.debug("Setting userName to {}", userName); + DummyAuthPluginImpl.userName = userName; + } + + public static synchronized String getUserName() { + return userName; + } +} http://git-wip-us.apache.org/repos/asf/sentry/blob/e62fa28d/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/ModifiableUserAuthenticationFilter.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/ModifiableUserAuthenticationFilter.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/ModifiableUserAuthenticationFilter.java deleted file mode 100644 index ac676a8..0000000 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/ModifiableUserAuthenticationFilter.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.tests.e2e.solr; - -import java.io.IOException; - -import javax.servlet.Filter; -import javax.servlet.FilterChain; -import javax.servlet.FilterConfig; -import javax.servlet.ServletException; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; -import javax.servlet.http.HttpServletRequest; - -import org.apache.solr.servlet.SolrRequestParsers; - -/** - * Authentication Filter that authenticates any request as user "junit" - */ -public class ModifiableUserAuthenticationFilter implements Filter { - - /** - * String that saves the user to be authenticated into Solr - */ - private static String userName = "admin"; - - @Override - public void init(FilterConfig filterConfig) throws ServletException { - SolrRequestParsers.DEFAULT.setAddRequestHeadersToContext(true); - } - - @Override - public void destroy() { - } - - @Override - public void doFilter(ServletRequest request, ServletResponse response, - FilterChain chain) throws IOException, ServletException { - HttpServletRequest httpRequest = (HttpServletRequest) request; - httpRequest.setAttribute("solr.user.name", userName); - chain.doFilter(request, response); - } - - /** - * Function to set the userName with the corresponding user passed as parameter - * @param solrUser - */ - public static void setUser(String solrUser) { - userName = solrUser; - } - - /** - * Function to return the authenticated user name defined. - * @param solrUser - */ - public static String getUser() { - return userName; - } -} http://git-wip-us.apache.org/repos/asf/sentry/blob/e62fa28d/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestCollAdminCoreOperations.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestCollAdminCoreOperations.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestCollAdminCoreOperations.java deleted file mode 100644 index b0d6db1..0000000 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestCollAdminCoreOperations.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sentry.tests.e2e.solr; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Random; - -import org.apache.solr.common.params.CollectionParams.CollectionAction; - -import org.junit.Test; -import static org.junit.Assert.assertEquals; - -public class TestCollAdminCoreOperations extends AbstractSolrSentryTestBase { - - private static final Logger LOG = LoggerFactory - .getLogger(TestCollAdminCoreOperations.class); - private static final String ADMIN_COLLECTION_NAME = "admin"; - private static final String TEST_COLLECTION_NAME = "sentryCollection"; - private static final List<Boolean> BOOLEAN_VALUES = Arrays.asList(new Boolean[]{true, false}); - private static final String DEFAULT_COLLECTION = "collection1"; - - /** - * Maximum number of combinations that will be tested by this class. - */ - private static final int MAX_TEST_RUNS = 64; - - /** - * Default number of combinations to be tested:15. - */ - private static int NUM_TESTS_TO_RUN = 15; - - @Test - public void testCollAdminCoreOperations() throws Exception { - String maxTestsToRun = System.getProperty("sentry.solr.e2e.maxTestsToRun"); - if (maxTestsToRun != null) { - if (maxTestsToRun.compareToIgnoreCase("all") == 0) { - NUM_TESTS_TO_RUN = MAX_TEST_RUNS; - } else { - NUM_TESTS_TO_RUN = Integer.parseInt(maxTestsToRun); - if (NUM_TESTS_TO_RUN > MAX_TEST_RUNS) { - NUM_TESTS_TO_RUN = MAX_TEST_RUNS; - } - } - } - - Random randomNum = new Random(); - HashSet<Integer> iterationSet = new HashSet<Integer>(); - while (iterationSet.size() < NUM_TESTS_TO_RUN) { - iterationSet.add(randomNum.nextInt(MAX_TEST_RUNS)); - } - int testCounter = 0; - - ArrayList<String> testFailures = new ArrayList<String>(); - // Upload configs to ZK - uploadConfigDirToZk(RESOURCES_DIR + File.separator + DEFAULT_COLLECTION - + File.separator + "conf"); - for (boolean admin_query : BOOLEAN_VALUES) { - for (boolean admin_update : BOOLEAN_VALUES) { - for (boolean admin_all : BOOLEAN_VALUES) { - String admin_test_user = getUsernameForPermissions(ADMIN_COLLECTION_NAME, admin_query, admin_update, admin_all); - - for (boolean coll_query : BOOLEAN_VALUES) { - for (boolean coll_update : BOOLEAN_VALUES) { - for (boolean coll_all : BOOLEAN_VALUES) { - if (!iterationSet.contains(testCounter)) { - testCounter = testCounter + 1; - continue; - } - testCounter = testCounter + 1; - - String coll_test_user = null; - try { - coll_test_user = admin_test_user - .concat("__") - .concat(getUsernameForPermissions(TEST_COLLECTION_NAME, coll_query, coll_update, coll_all)); - LOG.info("TEST_USER: " + coll_test_user); - - // Setup the environment - deleteCollection(TEST_COLLECTION_NAME); - - if ((admin_all || admin_update) && (coll_all || coll_update)) { - verifyCollectionAdminOpPass(coll_test_user, - CollectionAction.CREATE, - TEST_COLLECTION_NAME); - verifyCollectionAdminOpPass(coll_test_user, - CollectionAction.RELOAD, - TEST_COLLECTION_NAME); - verifyCollectionAdminOpPass(coll_test_user, - CollectionAction.DELETE, - TEST_COLLECTION_NAME); - } else { - verifyCollectionAdminOpFail(coll_test_user, - CollectionAction.CREATE, - TEST_COLLECTION_NAME); - // In-order to test RELOAD, DELETE for the current user, - // we need to setup a collection. - setupCollection(TEST_COLLECTION_NAME); - verifyCollectionAdminOpFail(coll_test_user, - CollectionAction.RELOAD, - TEST_COLLECTION_NAME); - verifyCollectionAdminOpFail(coll_test_user, - CollectionAction.DELETE, - TEST_COLLECTION_NAME); - } - } catch (Throwable testException) { - StringWriter stringWriter = new StringWriter(); - PrintWriter printWriter = new PrintWriter(stringWriter); - testException.printStackTrace(printWriter); - testFailures.add("\n\nTestFailure: User -> " + coll_test_user + "\n" - + stringWriter.toString()); - } - } - } - } - } - } - } - - assertEquals("Total test failures: " + testFailures.size() + " \n\n" - + testFailures.toString() + "\n\n\n", 0, testFailures.size()); - } -}
