SENTRY-186: e2e tests for solr document-level security (Gregory Chanan via Vamsee Yarlagadda)
Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/b4c39a1e Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/b4c39a1e Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/b4c39a1e Branch: refs/heads/master Commit: b4c39a1e89bd1f640d01e31d83d4833602e17a25 Parents: 574a685 Author: Vamsee <[email protected]> Authored: Tue Apr 29 13:56:44 2014 -0700 Committer: Vamsee <[email protected]> Committed: Tue Apr 29 13:56:44 2014 -0700 ---------------------------------------------------------------------- .../e2e/solr/AbstractSolrSentryTestBase.java | 20 +- .../tests/e2e/solr/TestDocLevelOperations.java | 289 +++ .../resources/solr/collection1/conf/schema.xml | 2 + .../collection1/conf/solrconfig-doclevel.xml | 1881 ++++++++++++++++++ .../solr/sentry/test-authz-provider.ini | 7 +- 5 files changed, 2193 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/b4c39a1e/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java index d58f3b9..e90891d 100644 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java @@ -75,6 +75,7 @@ public class AbstractSolrSentryTestBase { protected static final String ALL_DOCS = "*:*"; protected static final Random RANDOM = new Random(); protected static final String RESOURCES_DIR = "target" + File.separator + "test-classes" + File.separator + "solr"; + protected static final String CONF_DIR_IN_ZK = "conf1"; private static final int NUM_SERVERS = 4; private static void addPropertyToSentry(StringBuilder builder, String name, String value) { @@ -164,7 +165,7 @@ public class AbstractSolrSentryTestBase { * @return - the username as String * @throws Exception */ - private String getAuthenticatedUser() throws Exception { + protected String getAuthenticatedUser() throws Exception { return ModifiableUserAuthenticationFilter.getUser(); } @@ -675,12 +676,23 @@ public class AbstractSolrSentryTestBase { verifyUpdatePass(ADMIN_USER, collectionName, solrInputDoc); } - protected void uploadConfigDirToZk(String collectionConfigDir) throws Exception { + private ZkController getZkController() { SolrDispatchFilter dispatchFilter = (SolrDispatchFilter) miniSolrCloudCluster.getJettySolrRunners().get(0).getDispatchFilter().getFilter(); - ZkController zkController = dispatchFilter.getCores().getZkController(); + return dispatchFilter.getCores().getZkController(); + } + + protected void uploadConfigDirToZk(String collectionConfigDir) throws Exception { + ZkController zkController = getZkController(); // conf1 is the config used by AbstractFullDistribZkTestBase - zkController.uploadConfigDir(new File(collectionConfigDir), "conf1"); + zkController.uploadConfigDir(new File(collectionConfigDir), + CONF_DIR_IN_ZK); + } + + protected void uploadConfigFileToZk(String file, String nameInZk) throws Exception { + ZkController zkController = getZkController(); + zkController.getZkClient().makePath(ZkController.CONFIGS_ZKNODE + "/" + + CONF_DIR_IN_ZK + "/" + nameInZk, new File(file), false, true); } protected CloudSolrServer createNewCloudSolrServer() throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/b4c39a1e/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java new file mode 100644 index 0000000..eccc576 --- /dev/null +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.tests.e2e.solr; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.junit.After; +import org.junit.Before; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.commons.io.IOUtils; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.util.EntityUtils; + +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.impl.CloudSolrServer; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrInputDocument; + +import java.io.File; +import java.io.InputStream; +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Set; + +import org.junit.Test; + +/** + * Test the document-level security features + */ +public class TestDocLevelOperations extends AbstractSolrSentryTestBase { + private static final Logger LOG = LoggerFactory + .getLogger(TestDocLevelOperations.class); + private static final String DEFAULT_COLLECTION = "collection1"; + private static final String AUTH_FIELD = "sentry_auth"; + private static final int NUM_DOCS = 100; + private static final int EXTRA_AUTH_FIELDS = 2; + private String userName = null; + + @Before + public void beforeTest() throws Exception { + userName = getAuthenticatedUser(); + } + + @After + public void afterTest() throws Exception { + setAuthenticationUser(userName); + } + + private void setupCollectionWithDocSecurity(String name) throws Exception { + String configDir = RESOURCES_DIR + File.separator + DEFAULT_COLLECTION + + File.separator + "conf"; + uploadConfigDirToZk(configDir); + // replace solrconfig.xml with solrconfig-doc-level.xml + uploadConfigFileToZk(configDir + File.separator + "solrconfig-doclevel.xml", + "solrconfig.xml"); + setupCollection(name); + } + + private String doHttpGet(CloudSolrServer server, String path) throws Exception { + HttpClient httpClient = server.getLbServer().getHttpClient(); + Set<String> liveNodes = + server.getZkStateReader().getClusterState().getLiveNodes(); + assertTrue("Expected at least one live node", !liveNodes.isEmpty()); + String firstServer = liveNodes.toArray(new String[0])[0].replace("_solr", "/solr"); + URI uri = new URI("http://"; + firstServer + path); + HttpGet get = new HttpGet(uri); + HttpEntity httpEntity = null; + boolean success = false; + String retValue = ""; + try { + final HttpResponse response = httpClient.execute(get); + int httpStatus = response.getStatusLine().getStatusCode(); + httpEntity = response.getEntity(); + + if (httpEntity != null) { + InputStream is = httpEntity.getContent(); + ByteArrayOutputStream os = new ByteArrayOutputStream(); + try { + IOUtils.copyLarge(is, os); + os.flush(); + } finally { + IOUtils.closeQuietly(os); + IOUtils.closeQuietly(is); + } + retValue = os.toString(); + } + success = true; + } finally { + if (!success) { + EntityUtils.consumeQuietly(httpEntity); + get.abort(); + } + } + return retValue; + } + + /** + * Test that queries from different users only return the documents they have access to. + */ + @Test + public void testDocLevelOperations() throws Exception { + String collectionName = "docLevelCollection"; + setupCollectionWithDocSecurity(collectionName); + + // create documents + ArrayList<SolrInputDocument> docs = new ArrayList<SolrInputDocument>(); + for (int i = 0; i < NUM_DOCS; ++i) { + SolrInputDocument doc = new SolrInputDocument(); + String iStr = Long.toString(i); + doc.addField("id", iStr); + doc.addField("description", "description" + iStr); + + // put some bogus tokens in + for (int k = 0; k < EXTRA_AUTH_FIELDS; ++k) { + doc.addField(AUTH_FIELD, AUTH_FIELD + Long.toString(k)); + } + // 50% of docs get "junit", 50% get "admin" as token + if (i % 2 == 0) { + doc.addField(AUTH_FIELD, "junit"); + } else { + doc.addField(AUTH_FIELD, "admin"); + } + // add a token to all docs so we can check that we can get all + // documents returned + doc.addField(AUTH_FIELD, "docLevel"); + + docs.add(doc); + } + CloudSolrServer server = getCloudSolrServer(collectionName); + try { + server.add(docs); + server.commit(true, true); + + // queries + SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + + // as junit -- should get half the documents + setAuthenticationUser("junit"); + QueryResponse rsp = server.query(query); + SolrDocumentList docList = rsp.getResults(); + assertEquals(NUM_DOCS / 2, docList.getNumFound()); + for (SolrDocument doc : docList) { + String id = doc.getFieldValue("id").toString(); + assertEquals(0, Long.valueOf(id) % 2); + } + + // as admin -- should get the other half + setAuthenticationUser("admin"); + rsp = server.query(query); + docList = rsp.getResults(); + assertEquals(NUM_DOCS / 2, docList.getNumFound()); + for (SolrDocument doc : docList) { + String id = doc.getFieldValue("id").toString(); + assertEquals(1, Long.valueOf(id) % 2); + } + + // as docLevel -- should get all + setAuthenticationUser("docLevel"); + rsp = server.query(query); + assertEquals(NUM_DOCS, rsp.getResults().getNumFound()); + + // test filter queries work as AND -- i.e. user can't avoid doc-level + // checks by prefixing their own filterQuery + setAuthenticationUser("junit"); + String fq = URLEncoder.encode(" {!raw f=" + AUTH_FIELD + " v=docLevel}"); + String path = "/" + collectionName + "/select?q=*:*&fq="+fq; + String retValue = doHttpGet(server, path); + assertTrue(retValue.contains("numFound=\"" + NUM_DOCS / 2 + "\" ")); + + // test that user can't inject an "OR" into the query + final String syntaxErrorMsg = "org.apache.solr.search.SyntaxError: Cannot parse"; + fq = URLEncoder.encode(" {!raw f=" + AUTH_FIELD + " v=docLevel} OR "); + path = "/" + collectionName + "/select?q=*:*&fq="+fq; + retValue = doHttpGet(server, path); + assertTrue(retValue.contains(syntaxErrorMsg)); + + // same test, prefix OR this time + fq = URLEncoder.encode(" OR {!raw f=" + AUTH_FIELD + " v=docLevel}"); + path = "/" + collectionName + "/select?q=*:*&fq="+fq; + retValue = doHttpGet(server, path); + assertTrue(retValue.contains(syntaxErrorMsg)); + } finally { + server.shutdown(); + } + } + + /** + * Test the allGroupsToken. Make it a keyword in the query language ("OR") + * to make sure it is treated literally rather than interpreted. + */ + @Test + public void testAllGroupsToken() throws Exception { + String allGroupsToken = "OR"; + String collectionName = "allGroupsCollection"; + setupCollectionWithDocSecurity(collectionName); + + int junitFactor = 2; + int allGroupsFactor = 5; + + int totalJunitAdded = 0; // total docs added with junit token + int totalAllGroupsAdded = 0; // total number of docs with the allGroupsToken + int totalOnlyAllGroupsAdded = 0; // total number of docs with _only_ the allGroupsToken + + // create documents + ArrayList<SolrInputDocument> docs = new ArrayList<SolrInputDocument>(); + for (int i = 0; i < NUM_DOCS; ++i) { + boolean addedViaJunit = false; + SolrInputDocument doc = new SolrInputDocument(); + String iStr = Long.toString(i); + doc.addField("id", iStr); + doc.addField("description", "description" + iStr); + + if (i % junitFactor == 0) { + doc.addField(AUTH_FIELD, "junit"); + addedViaJunit = true; + ++totalJunitAdded; + } if (i % allGroupsFactor == 0) { + doc.addField(AUTH_FIELD, allGroupsToken); + ++totalAllGroupsAdded; + if (!addedViaJunit) ++totalOnlyAllGroupsAdded; + } + docs.add(doc); + } + // make sure our factors give us interesting results -- + // that some docs only have all groups and some only have junit + assert(totalOnlyAllGroupsAdded > 0); + assert(totalJunitAdded > totalAllGroupsAdded); + + CloudSolrServer server = getCloudSolrServer(collectionName); + try { + server.add(docs); + server.commit(true, true); + + // queries + SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + + // as admin -- should only get all groups token documents + setAuthenticationUser("admin"); + QueryResponse rsp = server.query(query); + SolrDocumentList docList = rsp.getResults(); + assertEquals(totalAllGroupsAdded, docList.getNumFound()); + for (SolrDocument doc : docList) { + String id = doc.getFieldValue("id").toString(); + assertEquals(0, Long.valueOf(id) % allGroupsFactor); + } + + // as junit -- should get junit added + onlyAllGroupsAdded + setAuthenticationUser("junit"); + rsp = server.query(query); + docList = rsp.getResults(); + assertEquals(totalJunitAdded + totalOnlyAllGroupsAdded, docList.getNumFound()); + for (SolrDocument doc : docList) { + String id = doc.getFieldValue("id").toString(); + boolean addedJunit = (Long.valueOf(id) % junitFactor) == 0; + boolean onlyAllGroups = !addedJunit && (Long.valueOf(id) % allGroupsFactor) == 0; + assertEquals(true, addedJunit || onlyAllGroups); + } + } finally { + server.shutdown(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/b4c39a1e/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml b/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml index 2aa68dd..66449ff 100644 --- a/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml +++ b/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml @@ -215,6 +215,8 @@ <dynamicField name="*_pi" type="pint" indexed="true" stored="true"/> <dynamicField name="*_c" type="currency" indexed="true" stored="true"/> + <dynamicField name="*_auth" type="string" indexed="true" stored="false" multiValued="true"/> + <dynamicField name="ignored_*" type="ignored" multiValued="true"/> <dynamicField name="attr_*" type="text_general" indexed="true" stored="true" multiValued="true"/>
