Repository: incubator-sentry Updated Branches: refs/heads/master 03b2f1882 -> 59bbfdc79
SENTRY-989: RealTimeGet with explicit ids can bypass document level authorization (Gregory Chanan, reviewed by Hao Hao) Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/59bbfdc7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/59bbfdc7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/59bbfdc7 Branch: refs/heads/master Commit: 59bbfdc7983c5aec465b643d2584ad888432e48b Parents: 03b2f18 Author: Gregory Chanan <gcha...@cloudera.com> Authored: Thu Feb 18 16:20:52 2016 -0800 Committer: Gregory Chanan <gcha...@cloudera.com> Committed: Mon Feb 22 14:24:16 2016 -0800 ---------------------------------------------------------------------- .../solr/handler/SecureRealTimeGetHandler.java | 36 ++ .../QueryDocAuthorizationComponent.java | 55 ++- .../component/SecureRealTimeGetComponent.java | 356 ++++++++++++++ .../e2e/solr/AbstractSolrSentryTestBase.java | 76 ++- .../tests/e2e/solr/DocLevelGenerator.java | 72 +++ .../tests/e2e/solr/TestDocLevelOperations.java | 237 ++++----- .../sentry/tests/e2e/solr/TestRealTimeGet.java | 476 +++++++++++++++++++ .../resources/solr/collection1/conf/schema.xml | 1 + .../collection1/conf/solrconfig-doclevel.xml | 19 +- .../solr/sentry/test-authz-provider.ini | 2 +- 10 files changed, 1197 insertions(+), 133 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/59bbfdc7/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/SecureRealTimeGetHandler.java ---------------------------------------------------------------------- diff --git a/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/SecureRealTimeGetHandler.java b/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/SecureRealTimeGetHandler.java new file mode 100644 index 0000000..db182ef --- /dev/null +++ b/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/SecureRealTimeGetHandler.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler; + + +import org.apache.solr.handler.component.RealTimeGetComponent; +import org.apache.solr.handler.component.SecureRealTimeGetComponent; + +import java.util.ArrayList; +import java.util.List; + +public class SecureRealTimeGetHandler extends RealTimeGetHandler { + @Override + protected List<String> getDefaultComponents() + { + List<String> names = new ArrayList<>(1); + names.add(RealTimeGetComponent.COMPONENT_NAME); + names.add(SecureRealTimeGetComponent.COMPONENT_NAME); + return names; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/59bbfdc7/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/component/QueryDocAuthorizationComponent.java ---------------------------------------------------------------------- diff --git a/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/component/QueryDocAuthorizationComponent.java b/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/component/QueryDocAuthorizationComponent.java index 666c088..be46a85 100644 --- a/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/component/QueryDocAuthorizationComponent.java +++ b/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/component/QueryDocAuthorizationComponent.java @@ -17,6 +17,12 @@ package org.apache.solr.handler.component; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.solr.common.SolrException; @@ -69,6 +75,40 @@ public class QueryDocAuthorizationComponent extends SearchComponent .append(value).append("}"); } + public String getFilterQueryStr(Set<String> roles) { + if (roles != null && roles.size() > 0) { + StringBuilder builder = new StringBuilder(); + for (String role : roles) { + addRawClause(builder, authField, role); + } + if (allRolesToken != null && !allRolesToken.isEmpty()) { + addRawClause(builder, authField, allRolesToken); + } + return builder.toString(); + } + return null; + } + + private BooleanClause getBooleanClause(String authField, String value) { + Term t = new Term(authField, value); + return new BooleanClause(new TermQuery(t), BooleanClause.Occur.SHOULD); + } + + public Query getFilterQuery(Set<String> roles) { + if (roles != null && roles.size() > 0) { + BooleanQuery query = new BooleanQuery(); + for (String role : roles) { + query.add(getBooleanClause(authField, role)); + } + if (allRolesToken != null && !allRolesToken.isEmpty()) { + query.add(getBooleanClause(authField, allRolesToken)); + } + return query; + } + + return null; + } + @Override public void prepare(ResponseBuilder rb) throws IOException { if (!enabled) { @@ -82,16 +122,9 @@ public class QueryDocAuthorizationComponent extends SearchComponent } Set<String> roles = sentryInstance.getRoles(userName); if (roles != null && roles.size() > 0) { - StringBuilder builder = new StringBuilder(); - for (String role : roles) { - addRawClause(builder, authField, role); - } - if (allRolesToken != null && !allRolesToken.isEmpty()) { - addRawClause(builder, authField, allRolesToken); - } + String filterQuery = getFilterQueryStr(roles); ModifiableSolrParams newParams = new ModifiableSolrParams(rb.req.getParams()); - String result = builder.toString(); - newParams.add("fq", result); + newParams.add("fq", filterQuery); rb.req.setParams(newParams); } else { throw new SolrException(SolrException.ErrorCode.UNAUTHORIZED, @@ -113,4 +146,8 @@ public class QueryDocAuthorizationComponent extends SearchComponent public String getSource() { return "$URL$"; } + + public boolean getEnabled() { + return enabled; + } } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/59bbfdc7/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/component/SecureRealTimeGetComponent.java ---------------------------------------------------------------------- diff --git a/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/component/SecureRealTimeGetComponent.java b/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/component/SecureRealTimeGetComponent.java new file mode 100644 index 0000000..e692f54 --- /dev/null +++ b/sentry-solr/solr-sentry-handlers/src/main/java/org/apache/solr/handler/component/SecureRealTimeGetComponent.java @@ -0,0 +1,356 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.component; + +import org.apache.lucene.document.Document; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.document.Field; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.BytesRef; + +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.SolrCore; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.response.transform.DocTransformer; +import org.apache.solr.response.transform.DocTransformers; +import org.apache.solr.response.transform.TransformContext; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.SolrReturnFields; +import org.apache.solr.search.ReturnFields; +import org.apache.solr.sentry.SentryIndexAuthorizationSingleton; +import org.apache.solr.update.AddUpdateCommand; +import org.apache.solr.update.UpdateCommand; +import org.apache.solr.update.UpdateLog; +import org.apache.solr.util.RefCounted; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; + +public class SecureRealTimeGetComponent extends SearchComponent +{ + private static Logger log = + LoggerFactory.getLogger(SecureRealTimeGetComponent.class); + public static String ID_FIELD_NAME = "_reserved_sentry_id"; + public static final String COMPONENT_NAME = "secureGet"; + + private SentryIndexAuthorizationSingleton sentryInstance; + + public SecureRealTimeGetComponent() { + this(SentryIndexAuthorizationSingleton.getInstance()); + } + + @VisibleForTesting + public SecureRealTimeGetComponent(SentryIndexAuthorizationSingleton sentryInstance) { + super(); + this.sentryInstance = sentryInstance; + } + + @Override + public void prepare(ResponseBuilder rb) throws IOException { + QueryDocAuthorizationComponent docComponent = + (QueryDocAuthorizationComponent)rb.req.getCore().getSearchComponent("queryDocAuthorization"); + if (docComponent != null) { + String userName = sentryInstance.getUserName(rb.req); + String superUser = (System.getProperty("solr.authorization.superuser", "solr")); + // security is never applied to the super user; for example, if solr internally is using + // real time get for replica synchronization, we need to return all the documents. + if (docComponent.getEnabled() && !superUser.equals(userName)) { + Set<String> roles = sentryInstance.getRoles(userName); + if (roles != null && roles.size() > 0) { + SolrReturnFields savedReturnFields = (SolrReturnFields)rb.rsp.getReturnFields(); + if (savedReturnFields == null) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Not able to authorize request because ReturnFields is invalid: " + savedReturnFields); + } + DocTransformer savedTransformer = savedReturnFields.getTransformer(); + Query filterQuery = docComponent.getFilterQuery(roles); + if (filterQuery != null) { + SolrReturnFields solrReturnFields = new AddDocIdReturnFields(rb.req, savedTransformer, filterQuery); + rb.rsp.setReturnFields(solrReturnFields); + } else { + throw new SolrException(SolrException.ErrorCode.UNAUTHORIZED, + "Request from user: " + userName + + "rejected because filter query was unable to be generated"); + } + } else { + throw new SolrException(SolrException.ErrorCode.UNAUTHORIZED, + "Request from user: " + userName + + " rejected because user is not associated with any roles"); + } + } + } else { + throw new SolrException(SolrException.ErrorCode.UNAUTHORIZED, + "RealTimeGetRequest request " + + " rejected because \"queryDocAuthorization\" component not defined"); + } + } + + @Override + public void process(ResponseBuilder rb) throws IOException { + if (!(rb.rsp.getReturnFields() instanceof AddDocIdReturnFields)) { + log.info("Skipping application of SecureRealTimeGetComponent because " + + " return field wasn't applied in prepare phase"); + return; + } + + final SolrQueryResponse rsp = rb.rsp; + ResponseFormatDocs responseFormatDocs = getResponseFormatDocs(rsp); + if (responseFormatDocs == null) { + return; // no documents to check + } + final SolrDocumentList docList = responseFormatDocs.getDocList(); + final AddDocIdReturnFields addDocIdRf = (AddDocIdReturnFields)rb.rsp.getReturnFields(); + final Query filterQuery = addDocIdRf.getFilterQuery(); + final DocTransformer transformer = addDocIdRf.getOriginalTransformer(); + + // we replaced the original transfer in order to add the document id, reapply it here + // so return documents in the correct format. + if (transformer != null) { + TransformContext context = new TransformContext(); + context.req = rb.req; + transformer.setContext(context); + } + + SolrCore core = rb.req.getCore(); + UpdateLog ulog = core.getUpdateHandler().getUpdateLog(); + SchemaField idField = core.getLatestSchema().getUniqueKeyField(); + FieldType fieldType = idField.getType(); + boolean openedRealTimeSearcher = false; + RefCounted<SolrIndexSearcher> searcherHolder = core.getRealtimeSearcher(); + + SolrDocumentList docListToReturn = new SolrDocumentList(); + try { + SolrIndexSearcher searcher = searcherHolder.get(); + for (SolrDocument doc : docList) { + // -1 doc id indicates this value was read from log; we need to open + // a new real time searcher to run the filter query against + if (doc.get(ID_FIELD_NAME) == -1 && !openedRealTimeSearcher) { + searcherHolder.decref(); + // hack to clear ulog maps since we don't have + // openRealtimeSearcher API from SOLR-8436 + AddUpdateCommand cmd = new AddUpdateCommand(rb.req); + cmd.setFlags(UpdateCommand.REPLAY); + ulog.add(cmd, true); + + searcherHolder = core.getRealtimeSearcher(); + searcher = searcherHolder.get(); + openedRealTimeSearcher = true; + } + + int docid = getFilteredInternalDocId(doc, idField, fieldType, filterQuery, searcher); + if (docid < 0) continue; + Document luceneDocument = searcher.doc(docid); + SolrDocument newDoc = toSolrDoc(luceneDocument, core.getLatestSchema()); + if( transformer != null ) { + transformer.transform(newDoc, docid); + } + docListToReturn.add(newDoc); + } + } finally { + searcherHolder.decref(); + searcherHolder = null; + } + if (responseFormatDocs.getUseResponseField()) { + rsp.getValues().remove("response"); + docListToReturn.setNumFound(docListToReturn.size()); + rsp.add("response", docListToReturn); + } else { + rsp.getValues().remove("doc"); + rsp.add("doc", docListToReturn.size() > 0 ? docListToReturn.get(0) : null); + } + } + + private static SolrDocument toSolrDoc(Document doc, IndexSchema schema) { + SolrDocument out = new SolrDocument(); + for ( IndexableField f : doc.getFields() ) { + // Make sure multivalued fields are represented as lists + Object existing = out.get(f.name()); + if (existing == null) { + SchemaField sf = schema.getFieldOrNull(f.name()); + + // don't return copyField targets + if (sf != null && schema.isCopyFieldTarget(sf)) continue; + + if (sf != null && sf.multiValued()) { + List<Object> vals = new ArrayList<>(); + vals.add( f ); + out.setField( f.name(), vals ); + } + else{ + out.setField( f.name(), f ); + } + } + else { + out.addField( f.name(), f ); + } + } + return out; + } + + // get the response format to use and the documents to check + private static ResponseFormatDocs getResponseFormatDocs(final SolrQueryResponse rsp) { + SolrDocumentList docList = (SolrDocumentList)rsp.getValues().get("response"); + SolrDocument singleDoc = (SolrDocument)rsp.getValues().get("doc"); + if (docList == null && singleDoc == null) { + return null; // no documents to filter + } + if (docList != null && singleDoc != null) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Not able to filter secure reponse, RealTimeGet returned both a doc list and " + + "an individual document"); + } + final boolean useResponseField = docList != null; + if (docList == null) { + docList = new SolrDocumentList(); + docList.add(singleDoc); + } + return new ResponseFormatDocs(useResponseField, docList); + } + + /** + * @param doc SolrDocument to check + * @param idField field where the id is stored + * @param fieldType type of id field + * @param filterQuery Query to filter by + * @param searcher SolrIndexSearcher on which to apply the filter query + * @returns the internal docid, or -1 if doc is not found or doesn't match filter + */ + private static int getFilteredInternalDocId(SolrDocument doc, SchemaField idField, FieldType fieldType, + Query filterQuery, SolrIndexSearcher searcher) throws IOException { + int docid = -1; + Field f = (Field)doc.getFieldValue(idField.getName()); + String idStr = f.stringValue(); + BytesRef idBytes = new BytesRef(); + fieldType.readableToIndexed(idStr, idBytes); + // get the internal document id + long segAndId = searcher.lookupId(idBytes); + + // if docid is valid, run it through the filter + if (segAndId >= 0) { + int segid = (int) segAndId; + AtomicReaderContext ctx = searcher.getTopReaderContext().leaves().get((int) (segAndId >> 32)); + docid = segid + ctx.docBase; + Weight weight = filterQuery.createWeight(searcher); + Scorer scorer = weight.scorer(ctx, null); + if (scorer == null || segid != scorer.advance(segid)) { + // filter doesn't match. + docid = -1; + } + } + return docid; + } + + @Override + public String getDescription() { + return "Handle Query Document Authorization for RealTimeGet"; + } + + @Override + public String getSource() { + return "$URL$"; + } + + private static class ResponseFormatDocs { + private boolean useResponseField; + private SolrDocumentList docList; + + public ResponseFormatDocs(boolean useResponseField, SolrDocumentList docList) { + this.useResponseField = useResponseField; + this.docList = docList; + } + + public boolean getUseResponseField() { return useResponseField; } + public SolrDocumentList getDocList() { return docList; } + } + + // ReturnField that adds a transformer to store the document id + private static class AddDocIdReturnFields extends SolrReturnFields { + private DocTransformer transformer; + private DocTransformer originalTransformer; + private Query filterQuery; + + public AddDocIdReturnFields(SolrQueryRequest req, DocTransformer docTransformer, + Query filterQuery) { + super(req); + this.originalTransformer = docTransformer; + this.filterQuery = filterQuery; + final DocTransformers docTransformers = new DocTransformers(); + if (originalTransformer != null) docTransformers.addTransformer(originalTransformer); + docTransformers.addTransformer(new DocIdAugmenter(ID_FIELD_NAME)); + this.transformer = docTransformers; + } + + @Override + public DocTransformer getTransformer() { + return transformer; + } + + public DocTransformer getOriginalTransformer() { + return originalTransformer; + } + + public Query getFilterQuery() { + return filterQuery; + } + } + + // the Solr DocIdAugmenterFactory does not store negative doc ids; + // we do here. + private static class DocIdAugmenter extends DocTransformer + { + final String name; + + public DocIdAugmenter( String display ) + { + this.name = display; + } + + @Override + public String getName() + { + return name; + } + + @Override + public void transform(SolrDocument doc, int docid) { + doc.setField( name, docid ); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/59bbfdc7/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java index 2495a9e..3a2104a 100644 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java @@ -28,6 +28,7 @@ import java.io.InputStream; import java.net.MalformedURLException; import java.net.URI; import java.util.Comparator; +import java.util.Iterator; import java.util.Map; import java.util.Random; import java.util.Set; @@ -61,6 +62,7 @@ import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; @@ -90,6 +92,7 @@ public class AbstractSolrSentryTestBase { protected static final Random RANDOM = new Random(); protected static final String RESOURCES_DIR = "target" + File.separator + "test-classes" + File.separator + "solr"; protected static final String CONF_DIR_IN_ZK = "conf1"; + protected static final String DEFAULT_COLLECTION = "collection1"; protected static final int NUM_SERVERS = 4; private static void addPropertyToSentry(StringBuilder builder, String name, String value) { @@ -413,17 +416,30 @@ public class AbstractSolrSentryTestBase { * @param solrUserName - User authenticated into Solr * @param adminOp - Admin operation to be performed * @param collectionName - Name of the collection to be queried - * @param ignoreError - boolean to specify whether to ignore the error if any occurred. - * (We may need this attribute for running DELETE command on a collection which doesn't exist) * @throws Exception */ protected void verifyCollectionAdminOpPass(String solrUserName, CollectionAction adminOp, String collectionName) throws Exception { + verifyCollectionAdminOpPass(solrUserName, adminOp, collectionName, null); + } + + /** + * Method to validate collection Admin operation pass + * @param solrUserName - User authenticated into Solr + * @param adminOp - Admin operation to be performed + * @param collectionName - Name of the collection to be queried + * @param params - SolrParams to use + * @throws Exception + */ + protected void verifyCollectionAdminOpPass(String solrUserName, + CollectionAction adminOp, + String collectionName, + SolrParams params) throws Exception { String originalUser = getAuthenticatedUser(); try { setAuthenticationUser(solrUserName); - QueryRequest request = populateCollectionAdminParams(adminOp, collectionName); + QueryRequest request = populateCollectionAdminParams(adminOp, collectionName, params); CloudSolrServer solrServer = createNewCloudSolrServer(); try { NamedList<Object> result = solrServer.request(request); @@ -449,12 +465,27 @@ public class AbstractSolrSentryTestBase { protected void verifyCollectionAdminOpFail(String solrUserName, CollectionAction adminOp, String collectionName) throws Exception { + verifyCollectionAdminOpFail(solrUserName, adminOp, collectionName, null); + } + + /** + * Method to validate collection Admin operation fail + * @param solrUserName - User authenticated into Solr + * @param adminOp - Admin operation to be performed + * @param collectionName - Name of the collection to be queried + * @param params - SolrParams to use + * @throws Exception + */ + protected void verifyCollectionAdminOpFail(String solrUserName, + CollectionAction adminOp, + String collectionName, + SolrParams params) throws Exception { String originalUser = getAuthenticatedUser(); try { setAuthenticationUser(solrUserName); try { - QueryRequest request = populateCollectionAdminParams(adminOp, collectionName); + QueryRequest request = populateCollectionAdminParams(adminOp, collectionName, params); CloudSolrServer solrServer = createNewCloudSolrServer(); try { NamedList<Object> result = solrServer.request(request); @@ -483,7 +514,20 @@ public class AbstractSolrSentryTestBase { * @return - instance of QueryRequest. */ public QueryRequest populateCollectionAdminParams(CollectionAction adminOp, - String collectionName) { + String collectionName) { + return populateCollectionAdminParams(adminOp, collectionName, null); + } + + /** + * Method to populate the Solr params based on the collection admin being performed. + * @param adminOp - Collection admin operation + * @param collectionName - Name of the collection + * @param params - SolrParams to use + * @return - instance of QueryRequest. + */ + public QueryRequest populateCollectionAdminParams(CollectionAction adminOp, + String collectionName, + SolrParams params) { ModifiableSolrParams modParams = new ModifiableSolrParams(); modParams.set(CoreAdminParams.ACTION, adminOp.name()); switch (adminOp) { @@ -519,6 +563,14 @@ public class AbstractSolrSentryTestBase { throw new IllegalArgumentException("Admin operation: " + adminOp + " is not supported!"); } + if (params != null) { + Iterator<String> it = params.getParameterNamesIterator(); + while (it.hasNext()) { + String param = it.next(); + String [] value = params.getParams(param); + modParams.set(param, value); + } + } QueryRequest request = new QueryRequest(modParams); request.setPath("/admin/collections"); return request; @@ -701,16 +753,22 @@ public class AbstractSolrSentryTestBase { } protected void uploadConfigDirToZk(String collectionConfigDir) throws Exception { + uploadConfigDirToZk(collectionConfigDir, CONF_DIR_IN_ZK); + } + + protected void uploadConfigDirToZk(String collectionConfigDir, String confDirInZk) throws Exception { ZkController zkController = getZkController(); - // conf1 is the config used by AbstractFullDistribZkTestBase - zkController.uploadConfigDir(new File(collectionConfigDir), - CONF_DIR_IN_ZK); + zkController.uploadConfigDir(new File(collectionConfigDir), confDirInZk); } protected void uploadConfigFileToZk(String file, String nameInZk) throws Exception { + uploadConfigFileToZk(file, nameInZk, CONF_DIR_IN_ZK); + } + + protected void uploadConfigFileToZk(String file, String nameInZk, String confDirInZk) throws Exception { ZkController zkController = getZkController(); zkController.getZkClient().makePath(ZkController.CONFIGS_ZKNODE + "/" - + CONF_DIR_IN_ZK + "/" + nameInZk, new File(file), false, true); + + confDirInZk + "/" + nameInZk, new File(file), false, true); } protected CloudSolrServer createNewCloudSolrServer() throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/59bbfdc7/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/DocLevelGenerator.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/DocLevelGenerator.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/DocLevelGenerator.java new file mode 100644 index 0000000..30afd4c --- /dev/null +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/DocLevelGenerator.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.tests.e2e.solr; + +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.client.solrj.impl.CloudSolrServer; + +import java.util.ArrayList; + +public class DocLevelGenerator { + private String collection; + private String authField; + + public DocLevelGenerator(String collection, String authField) { + this.collection = collection; + this.authField = authField; + } + + /** + * Generates docs according to the following parameters: + * + * @param server SolrServer to use + * @param numDocs number of documents to generate + * @param evenDocsToken every even number doc gets this token added to the authField + * @param oddDocsToken every odd number doc gets this token added to the authField + * @param extraAuthFieldsCount generates this number of bogus entries in the authField + */ + public void generateDocs(CloudSolrServer server, int numDocs, String evenDocsToken, String oddDocsToken, int extraAuthFieldsCount) throws Exception { + + // create documents + ArrayList<SolrInputDocument> docs = new ArrayList<SolrInputDocument>(); + for (int i = 0; i < numDocs; ++i) { + SolrInputDocument doc = new SolrInputDocument(); + String iStr = Long.toString(i); + doc.addField("id", iStr); + doc.addField("description", "description" + iStr); + + // put some bogus tokens in + for (int k = 0; k < extraAuthFieldsCount; ++k) { + doc.addField(authField, authField + Long.toString(k)); + } + // even docs get evenDocsToken, odd docs get oddDocsToken + if (i % 2 == 0) { + doc.addField(authField, evenDocsToken); + } else { + doc.addField(authField, oddDocsToken); + } + // add a token to all docs so we can check that we can get all + // documents returned + doc.addField(authField, "docLevel_role"); + + docs.add(doc); + } + + server.add(docs); + server.commit(true, true); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/59bbfdc7/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java index ff508e1..46399df 100644 --- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java @@ -25,11 +25,15 @@ import static org.junit.Assert.assertTrue; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.impl.CloudSolrServer; +import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; import java.io.File; import java.net.URLEncoder; @@ -44,7 +48,6 @@ import org.junit.Test; public class TestDocLevelOperations extends AbstractSolrSentryTestBase { private static final Logger LOG = LoggerFactory .getLogger(TestDocLevelOperations.class); - private static final String DEFAULT_COLLECTION = "collection1"; private static final String AUTH_FIELD = "sentry_auth"; private static final int NUM_DOCS = 100; private static final int EXTRA_AUTH_FIELDS = 2; @@ -70,6 +73,31 @@ public class TestDocLevelOperations extends AbstractSolrSentryTestBase { setupCollection(name); } + private QueryRequest getRealTimeGetRequest() { + // real time get request + StringBuilder idsBuilder = new StringBuilder("0"); + for (int i = 1; i < NUM_DOCS; ++i) { + idsBuilder.append("," + i); + } + return getRealTimeGetRequest(idsBuilder.toString()); + } + + private QueryRequest getRealTimeGetRequest(String ids) { + final ModifiableSolrParams idsParams = new ModifiableSolrParams(); + idsParams.add("ids", ids); + return new QueryRequest() { + @Override + public String getPath() { + return "/get"; + } + + @Override + public SolrParams getParams() { + return idsParams; + } + }; + } + /** * Creates docs as follows and verifies queries work as expected: * - creates NUM_DOCS documents, where the document id equals the order @@ -84,67 +112,45 @@ public class TestDocLevelOperations extends AbstractSolrSentryTestBase { // ensure no current documents verifyDeletedocsPass(ADMIN_USER, collectionName, true); - // create documents - ArrayList<SolrInputDocument> docs = new ArrayList<SolrInputDocument>(); - for (int i = 0; i < NUM_DOCS; ++i) { - SolrInputDocument doc = new SolrInputDocument(); - String iStr = Long.toString(i); - doc.addField("id", iStr); - doc.addField("description", "description" + iStr); - - // put some bogus tokens in - for (int k = 0; k < EXTRA_AUTH_FIELDS; ++k) { - doc.addField(AUTH_FIELD, AUTH_FIELD + Long.toString(k)); - } - // 50% of docs get "junit", 50% get "admin" as token - if (i % 2 == 0) { - doc.addField(AUTH_FIELD, "junit_role"); - } else { - doc.addField(AUTH_FIELD, "admin_role"); - } - // add a token to all docs so we can check that we can get all - // documents returned - doc.addField(AUTH_FIELD, "docLevel_role"); - - docs.add(doc); - } CloudSolrServer server = getCloudSolrServer(collectionName); try { - server.add(docs); - server.commit(true, true); + DocLevelGenerator generator = new DocLevelGenerator(collectionName, AUTH_FIELD); + generator.generateDocs(server, NUM_DOCS, "junit_role", "admin_role", EXTRA_AUTH_FIELDS); - // queries - SolrQuery query = new SolrQuery(); - query.setQuery("*:*"); + querySimple(new QueryRequest(new SolrQuery("*:*")), server, checkNonAdminUsers); + querySimple(getRealTimeGetRequest(), server, checkNonAdminUsers); + } finally { + server.shutdown(); + } + } - // as admin -- should get the other half - setAuthenticationUser("admin"); - QueryResponse rsp = server.query(query); - SolrDocumentList docList = rsp.getResults(); + private void querySimple(QueryRequest request, CloudSolrServer server, + boolean checkNonAdminUsers) throws Exception { + // as admin -- should get the other half + setAuthenticationUser("admin"); + QueryResponse rsp = request.process(server); + SolrDocumentList docList = rsp.getResults(); + assertEquals(NUM_DOCS / 2, docList.getNumFound()); + for (SolrDocument doc : docList) { + String id = doc.getFieldValue("id").toString(); + assertEquals(1, Long.valueOf(id) % 2); + } + + if (checkNonAdminUsers) { + // as junit -- should get half the documents + setAuthenticationUser("junit"); + rsp = request.process(server); + docList = rsp.getResults(); assertEquals(NUM_DOCS / 2, docList.getNumFound()); for (SolrDocument doc : docList) { String id = doc.getFieldValue("id").toString(); - assertEquals(1, Long.valueOf(id) % 2); + assertEquals(0, Long.valueOf(id) % 2); } - if (checkNonAdminUsers) { - // as junit -- should get half the documents - setAuthenticationUser("junit"); - rsp = server.query(query); - docList = rsp.getResults(); - assertEquals(NUM_DOCS / 2, docList.getNumFound()); - for (SolrDocument doc : docList) { - String id = doc.getFieldValue("id").toString(); - assertEquals(0, Long.valueOf(id) % 2); - } - - // as docLevel -- should get all - setAuthenticationUser("docLevel"); - rsp = server.query(query); - assertEquals(NUM_DOCS, rsp.getResults().getNumFound()); - } - } finally { - server.shutdown(); + // as docLevel -- should get all + setAuthenticationUser("docLevel"); + rsp = request.process(server); + assertEquals(NUM_DOCS, rsp.getResults().getNumFound()); } } @@ -237,31 +243,10 @@ public class TestDocLevelOperations extends AbstractSolrSentryTestBase { server.add(docs); server.commit(true, true); - // queries - SolrQuery query = new SolrQuery(); - query.setQuery("*:*"); - - // as admin -- should only get all roles token documents - setAuthenticationUser("admin"); - QueryResponse rsp = server.query(query); - SolrDocumentList docList = rsp.getResults(); - assertEquals(totalAllRolesAdded, docList.getNumFound()); - for (SolrDocument doc : docList) { - String id = doc.getFieldValue("id").toString(); - assertEquals(0, Long.valueOf(id) % allRolesFactor); - } - - // as junit -- should get junit added + onlyAllRolesAdded - setAuthenticationUser("junit"); - rsp = server.query(query); - docList = rsp.getResults(); - assertEquals(totalJunitAdded + totalOnlyAllRolesAdded, docList.getNumFound()); - for (SolrDocument doc : docList) { - String id = doc.getFieldValue("id").toString(); - boolean addedJunit = (Long.valueOf(id) % junitFactor) == 0; - boolean onlyAllRoles = !addedJunit && (Long.valueOf(id) % allRolesFactor) == 0; - assertEquals(true, addedJunit || onlyAllRoles); - } + checkAllRolesToken(new QueryRequest(new SolrQuery("*:*")), server, + totalAllRolesAdded, totalOnlyAllRolesAdded, allRolesFactor, totalJunitAdded, junitFactor); + checkAllRolesToken(getRealTimeGetRequest(), server, + totalAllRolesAdded, totalOnlyAllRolesAdded, allRolesFactor, totalJunitAdded, junitFactor); } finally { server.shutdown(); } @@ -270,6 +255,31 @@ public class TestDocLevelOperations extends AbstractSolrSentryTestBase { } } + private void checkAllRolesToken(QueryRequest request, CloudSolrServer server, + int totalAllRolesAdded, int totalOnlyAllRolesAdded, int allRolesFactor, int totalJunitAdded, int junitFactor) throws Exception { + // as admin -- should only get all roles token documents + setAuthenticationUser("admin"); + QueryResponse rsp = request.process(server); + SolrDocumentList docList = rsp.getResults(); + assertEquals(totalAllRolesAdded, docList.getNumFound()); + for (SolrDocument doc : docList) { + String id = doc.getFieldValue("id").toString(); + assertEquals(0, Long.valueOf(id) % allRolesFactor); + } + + // as junit -- should get junit added + onlyAllRolesAdded + setAuthenticationUser("junit"); + rsp = request.process(server); + docList = rsp.getResults(); + assertEquals(totalJunitAdded + totalOnlyAllRolesAdded, docList.getNumFound()); + for (SolrDocument doc : docList) { + String id = doc.getFieldValue("id").toString(); + boolean addedJunit = (Long.valueOf(id) % junitFactor) == 0; + boolean onlyAllRoles = !addedJunit && (Long.valueOf(id) % allRolesFactor) == 0; + assertEquals(true, addedJunit || onlyAllRoles); + } + } + /** * delete the docs as "deleteUser" using deleteByQuery "deleteQueryStr". * Verify that number of docs returned for "queryUser" equals @@ -280,32 +290,35 @@ public class TestDocLevelOperations extends AbstractSolrSentryTestBase { createDocsAndQuerySimple(collectionName, true); CloudSolrServer server = getCloudSolrServer(collectionName); try { - SolrQuery query = new SolrQuery(); - query.setQuery("*:*"); - setAuthenticationUser(deleteUser); server.deleteByQuery(deleteByQueryStr); server.commit(); - QueryResponse rsp = server.query(query); - long junitResults = rsp.getResults().getNumFound(); - assertEquals(0, junitResults); - - setAuthenticationUser(queryUser); - rsp = server.query(query); - long docLevelResults = rsp.getResults().getNumFound(); - assertEquals(expectedQueryDocs, docLevelResults); + + checkDeleteByQuery(new QueryRequest(new SolrQuery("*:*")), server, + queryUser, expectedQueryDocs); + checkDeleteByQuery(getRealTimeGetRequest(), server, + queryUser, expectedQueryDocs); } finally { server.shutdown(); } } + private void checkDeleteByQuery(QueryRequest query, CloudSolrServer server, + String queryUser, int expectedQueryDocs) throws Exception { + QueryResponse rsp = query.process(server); + long junitResults = rsp.getResults().getNumFound(); + assertEquals(0, junitResults); + + setAuthenticationUser(queryUser); + rsp = query.process(server); + long docLevelResults = rsp.getResults().getNumFound(); + assertEquals(expectedQueryDocs, docLevelResults); + } + private void deleteByIdTest(String collectionName) throws Exception { createDocsAndQuerySimple(collectionName, true); CloudSolrServer server = getCloudSolrServer(collectionName); try { - SolrQuery query = new SolrQuery(); - query.setQuery("*:*"); - setAuthenticationUser("junit"); List<String> allIds = new ArrayList<String>(NUM_DOCS); for (int i = 0; i < NUM_DOCS; ++i) { @@ -314,19 +327,25 @@ public class TestDocLevelOperations extends AbstractSolrSentryTestBase { server.deleteById(allIds); server.commit(); - QueryResponse rsp = server.query(query); - long junitResults = rsp.getResults().getNumFound(); - assertEquals(0, junitResults); - - setAuthenticationUser("docLevel"); - rsp = server.query(query); - long docLevelResults = rsp.getResults().getNumFound(); - assertEquals(0, docLevelResults); + checkDeleteById(new QueryRequest(new SolrQuery("*:*")), server); + checkDeleteById(getRealTimeGetRequest(), server); } finally { server.shutdown(); } } + private void checkDeleteById(QueryRequest request, CloudSolrServer server) + throws Exception { + QueryResponse rsp = request.process(server); + long junitResults = rsp.getResults().getNumFound(); + assertEquals(0, junitResults); + + setAuthenticationUser("docLevel"); + rsp = request.process(server); + long docLevelResults = rsp.getResults().getNumFound(); + assertEquals(0, docLevelResults); + } + private void updateDocsTest(String collectionName) throws Exception { createDocsAndQuerySimple(collectionName, true); CloudSolrServer server = getCloudSolrServer(collectionName); @@ -335,10 +354,10 @@ public class TestDocLevelOperations extends AbstractSolrSentryTestBase { String docIdStr = Long.toString(1); // verify we can't view one of the odd documents - SolrQuery query = new SolrQuery(); - query.setQuery("id:"+docIdStr); - QueryResponse rsp = server.query(query); - assertEquals(0, rsp.getResults().getNumFound()); + QueryRequest query = new QueryRequest(new SolrQuery("id:"+docIdStr)); + QueryRequest rtgQuery = getRealTimeGetRequest(docIdStr); + checkUpdateDocsQuery(query, server, 0); + checkUpdateDocsQuery(rtgQuery, server, 0); // overwrite the document that we can't see ArrayList<SolrInputDocument> docs = new ArrayList<SolrInputDocument>(); @@ -351,13 +370,19 @@ public class TestDocLevelOperations extends AbstractSolrSentryTestBase { server.commit(); // verify we can now view the document - rsp = server.query(query); - assertEquals(1, rsp.getResults().getNumFound()); + checkUpdateDocsQuery(query, server, 1); + checkUpdateDocsQuery(rtgQuery, server, 1); } finally { server.shutdown(); } } + private void checkUpdateDocsQuery(QueryRequest request, CloudSolrServer server, int expectedDocs) + throws Exception { + QueryResponse rsp = request.process(server); + assertEquals(expectedDocs, rsp.getResults().getNumFound()); + } + @Test public void testUpdateDeleteOperations() throws Exception { String collectionName = "testUpdateDeleteOperations"; http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/59bbfdc7/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestRealTimeGet.java ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestRealTimeGet.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestRealTimeGet.java new file mode 100644 index 0000000..0d25562 --- /dev/null +++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestRealTimeGet.java @@ -0,0 +1,476 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sentry.tests.e2e.solr; + +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrServer; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CollectionParams.CollectionAction; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Random; +import java.util.Set; + +public class TestRealTimeGet extends AbstractSolrSentryTestBase { + private static final Logger LOG = LoggerFactory + .getLogger(TestRealTimeGet.class); + private static final String AUTH_FIELD = "sentry_auth"; + private static final Random rand = new Random(); + private String userName = null; + + @Before + public void beforeTest() throws Exception { + userName = getAuthenticatedUser(); + } + + @After + public void afterTest() throws Exception { + setAuthenticationUser(userName); + } + + private void setupCollectionWithDocSecurity(String name) throws Exception { + setupCollectionWithDocSecurity(name, 2); + } + + private void setupCollectionWithDocSecurity(String name, int shards) throws Exception { + String configDir = RESOURCES_DIR + File.separator + DEFAULT_COLLECTION + + File.separator + "conf"; + uploadConfigDirToZk(configDir, name); + // replace solrconfig.xml with solrconfig-doc-level.xml + uploadConfigFileToZk(configDir + File.separator + "solrconfig-doclevel.xml", + "solrconfig.xml", name); + ModifiableSolrParams modParams = new ModifiableSolrParams(); + modParams.set("numShards", shards); + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < shards; ++i) { + if (i != 0) builder.append(","); + builder.append("shard").append(i+1); + } + modParams.set("shards", builder.toString()); + verifyCollectionAdminOpPass(ADMIN_USER, CollectionAction.CREATE, name, modParams); + } + + private void setupCollectionWithoutDocSecurity(String name) throws Exception { + String configDir = RESOURCES_DIR + File.separator + DEFAULT_COLLECTION + + File.separator + "conf"; + uploadConfigDirToZk(configDir, name); + setupCollection(name); + } + + private QueryRequest getRealTimeGetRequest(final SolrParams params) { + return new QueryRequest() { + @Override + public String getPath() { + return "/get"; + } + + @Override + public SolrParams getParams() { + return params; + } + }; + } + + private void assertExpected(ExpectedResult expectedResult, QueryResponse rsp, + ExpectedResult controlExpectedResult, QueryResponse controlRsp) throws Exception { + SolrDocumentList docList = rsp.getResults(); + SolrDocumentList controlDocList = controlRsp.getResults(); + SolrDocument doc = (SolrDocument)rsp.getResponse().get("doc"); + SolrDocument controlDoc = (SolrDocument)controlRsp.getResponse().get("doc"); + + if (expectedResult.expectedDocs == 0) { + // could be null rather than 0 size, check against control that format is identical + assertNull("Should be no doc present: " + doc, doc); + assertNull("Should be no doc present: " + controlDoc, controlDoc); + assertTrue((docList == null && controlDocList == null) || + (controlDocList.getNumFound() == 0 && controlDocList.getNumFound() == 0)); + } else { + if (docList == null) { + assertNull(controlDocList); + assertNotNull(doc); + assertNotNull(controlDoc); + } else { + assertNotNull(controlDocList); + assertNull(doc); + assertNull(controlDoc); + assertEquals(expectedResult.expectedDocs, docList.getNumFound()); + assertEquals(docList.getNumFound(), controlDocList.getNumFound()); + } + } + } + + private QueryResponse getIdResponse(ExpectedResult expectedResult) throws Exception { + ModifiableSolrParams params = new ModifiableSolrParams(); + for (int i = 0; i < expectedResult.ids.length; ++i) { + params.add("id", expectedResult.ids[ i ]); + } + if (expectedResult.fl != null) { + params.add("fl", expectedResult.fl); + } + QueryRequest request = getRealTimeGetRequest(params); + return request.process(expectedResult.server); + } + + private QueryResponse getIdsResponse(ExpectedResult expectedResult) throws Exception { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < expectedResult.ids.length; ++i) { + if (i != 0) builder.append(","); + builder.append(expectedResult.ids[ i ]); + } + ModifiableSolrParams params = new ModifiableSolrParams(); + params.add("ids", builder.toString()); + if (expectedResult.fl != null) { + params.add("fl", expectedResult.fl); + } + QueryRequest request = getRealTimeGetRequest(params); + return request.process(expectedResult.server); + } + + private void assertIdVsIds(ExpectedResult expectedResult, ExpectedResult controlExpectedResult) + throws Exception { + // test specifying with "id" + QueryResponse idRsp = getIdResponse(expectedResult); + QueryResponse idControlRsp = getIdResponse(controlExpectedResult); + assertExpected(expectedResult, idRsp, controlExpectedResult, idControlRsp); + + // test specifying with "ids" + QueryResponse idsRsp = getIdsResponse(expectedResult); + QueryResponse idsControlRsp = getIdsResponse(controlExpectedResult); + assertExpected(expectedResult, idsRsp, controlExpectedResult, idsControlRsp); + } + + @Test + public void testIdvsIds() throws Exception { + final String collection = "testIdvsIds"; + final String collectionControl = collection + "Control"; + setupCollectionWithDocSecurity(collection); + setupCollectionWithoutDocSecurity(collectionControl); + CloudSolrServer server = getCloudSolrServer(collection); + CloudSolrServer serverControl = getCloudSolrServer(collectionControl); + + try { + for (CloudSolrServer s : new CloudSolrServer [] {server, serverControl}) { + DocLevelGenerator generator = new DocLevelGenerator(s.getDefaultCollection(), AUTH_FIELD); + generator.generateDocs(s, 100, "junit_role", "admin_role", 2); + } + + // check that control collection does not filter + assertIdVsIds(new ExpectedResult(serverControl, new String[] {"2"}, 1), + new ExpectedResult(serverControl, new String[] {"2"}, 1)); + + // single id + assertIdVsIds(new ExpectedResult(server, new String[] {"1"}, 1), + new ExpectedResult(serverControl, new String[] {"1"}, 1)); + + // single id (invalid) + assertIdVsIds(new ExpectedResult(server, new String[] {"bogusId"}, 0), + new ExpectedResult(serverControl, new String[] {"bogusId"}, 0)); + + // single id (no permission) + assertIdVsIds(new ExpectedResult(server, new String[] {"2"}, 0), + new ExpectedResult(serverControl, new String[] {"2fake"}, 0)); + + // multiple ids (some invalid, some valid, some no permission) + assertIdVsIds(new ExpectedResult(server, new String[] {"bogus1", "1", "2"}, 1), + new ExpectedResult(serverControl, new String[] {"bogus1", "1", "bogus2"}, 1)); + assertIdVsIds(new ExpectedResult(server, new String[] {"bogus1", "1", "2", "3"}, 2), + new ExpectedResult(serverControl, new String[] {"bogus1", "1", "bogus2", "3"}, 2)); + + // multiple ids (all invalid) + assertIdVsIds(new ExpectedResult(server, new String[] {"bogus1", "bogus2", "bogus3"}, 0), + new ExpectedResult(serverControl, new String[] {"bogus1", "bogus2", "bogus3"}, 0)); + + // multiple ids (all no permission) + assertIdVsIds(new ExpectedResult(server, new String[] {"2", "4", "6"}, 0), + new ExpectedResult(serverControl, new String[] {"bogus2", "bogus4", "bogus6"}, 0)); + + } finally { + server.shutdown(); + serverControl.shutdown(); + } + } + + private void assertFlOnDocList(SolrDocumentList list, Set<String> expectedIds, + List<String> expectedFields) { + assertEquals("Doc list size should be: " + expectedIds.size(), expectedIds.size(), list.getNumFound()); + for (SolrDocument doc : list) { + expectedIds.contains(doc.get("id")); + for (String field : expectedFields) { + assertNotNull("Field: " + field + " should not be null in doc: " + doc, doc.get(field)); + } + assertEquals("doc should have: " + expectedFields.size() + " fields. Doc: " + doc, + expectedFields.size(), doc.getFieldNames().size()); + } + } + + private void assertFl(CloudSolrServer server, String [] ids, Set<String> expectedIds, + String fl, List<String> expectedFields) throws Exception { + { + QueryResponse idRsp = getIdResponse(new ExpectedResult(server, ids, expectedIds.size(), fl)); + SolrDocumentList idList = idRsp.getResults(); + assertFlOnDocList(idList, expectedIds, expectedFields); + } + { + QueryResponse idsRsp = getIdsResponse(new ExpectedResult(server, ids, expectedIds.size(), fl)); + SolrDocumentList idsList = idsRsp.getResults(); + assertFlOnDocList(idsList, expectedIds, expectedFields); + } + } + + @Test + public void testFl() throws Exception { + final String collection = "testFl"; + // FixMe: have to use one shard, because of a Solr bug where "fl" is not applied to + // multi-shard get requests + setupCollectionWithDocSecurity(collection, 1); + CloudSolrServer server = getCloudSolrServer(collection); + + try { + DocLevelGenerator generator = new DocLevelGenerator(collection, AUTH_FIELD); + generator.generateDocs(server, 100, "junit_role", "admin_role", 2); + String [] ids = new String[] {"1", "3", "5"}; + + assertFl(server, ids, new HashSet<String>(Arrays.asList(ids)), "id", Arrays.asList("id")); + assertFl(server, ids, new HashSet<String>(Arrays.asList(ids)), null, Arrays.asList("id", "description", "_version_")); + // test transformer + assertFl(server, ids, new HashSet<String>(Arrays.asList(ids)), "id,mydescription:description", Arrays.asList("id", "mydescription")); + } finally { + server.shutdown(); + } + } + + @Test + public void testNonCommitted() throws Exception { + final String collection = "testNonCommitted"; + setupCollectionWithDocSecurity(collection, 1); + CloudSolrServer server = getCloudSolrServer(collection); + + try { + DocLevelGenerator generator = new DocLevelGenerator(collection, AUTH_FIELD); + generator.generateDocs(server, 100, "junit_role", "admin_role", 2); + + // make some uncommitted modifications and ensure they are reflected + server.deleteById("1"); + + SolrInputDocument doc2 = new SolrInputDocument(); + doc2.addField("id", "2"); + doc2.addField("description", "description2"); + doc2.addField(AUTH_FIELD, "admin_role"); + + SolrInputDocument doc3 = new SolrInputDocument(); + doc3.addField("id", "3"); + doc3.addField("description", "description3"); + doc3.addField(AUTH_FIELD, "junit_role"); + + SolrInputDocument doc200 = new SolrInputDocument(); + doc200.addField("id", "200"); + doc200.addField("description", "description200"); + doc200.addField(AUTH_FIELD, "admin_role"); + server.add(Arrays.asList(new SolrInputDocument [] {doc2, doc3, doc200})); + + assertFl(server, new String[] {"1", "2", "3", "4", "5", "200"}, + new HashSet<String>(Arrays.asList("2", "5", "200")), "id", Arrays.asList("id")); + } finally { + server.shutdown(); + } + } + + private void assertConcurrentOnDocList(SolrDocumentList list, String authField, String expectedAuthFieldValue) { + for (SolrDocument doc : list) { + Collection<Object> authFieldValues = doc.getFieldValues(authField); + assertNotNull(authField + " should not be null. Doc: " + doc, authFieldValues); + + boolean foundAuthFieldValue = false; + for (Object obj : authFieldValues) { + if (obj.toString().equals(expectedAuthFieldValue)) { + foundAuthFieldValue = true; + break; + } + } + assertTrue("Did not find: " + expectedAuthFieldValue + " in doc: " + doc, foundAuthFieldValue); + } + } + + private void assertConcurrent(CloudSolrServer server, String [] ids, String authField, String expectedAuthFieldValue) + throws Exception { + { + QueryResponse idRsp = getIdResponse(new ExpectedResult(server, ids, -1, null)); + SolrDocumentList idList = idRsp.getResults(); + assertConcurrentOnDocList(idList, authField, expectedAuthFieldValue); + } + { + QueryResponse idsRsp = getIdsResponse(new ExpectedResult(server, ids, -1, null)); + SolrDocumentList idsList = idsRsp.getResults(); + assertConcurrentOnDocList(idsList, authField, expectedAuthFieldValue); + } + } + + @Test + public void testConcurrentChanges() throws Exception { + final String collection = "testConcurrentChanges"; + // Ensure the auth field is stored so we can check a consistent doc is returned + final String authField = "sentry_auth_stored"; + System.setProperty("sentry.auth.field", authField); + setupCollectionWithDocSecurity(collection, 1); + CloudSolrServer server = getCloudSolrServer(collection); + int numQueries = 5; + + try { + DocLevelGenerator generator = new DocLevelGenerator(collection, authField); + generator.generateDocs(server, 100, "junit_role", "admin_role", 2); + + List<AuthFieldModifyThread> threads = new LinkedList<AuthFieldModifyThread>(); + int docsToModify = 10; + for (int i = 0; i < docsToModify; ++i) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", Integer.toString(i)); + doc.addField("description", "description" + Integer.toString(i)); + doc.addField(authField, "junit_role"); + server.add(doc); + + threads.add(new AuthFieldModifyThread(server, doc, + authField, "junit_role", "admin_role")); + } + server.commit(); + + for (AuthFieldModifyThread thread : threads) { + thread.start(); + } + + // query + String [] ids = new String[docsToModify]; + for (int j = 0; j < ids.length; ++j) { + ids[ j ] = Integer.toString(j); + } + for (int k = 0; k < numQueries; ++k) { + assertConcurrent(server, ids, authField, "admin_role"); + } + + for (AuthFieldModifyThread thread : threads) { + thread.setFinished(); + thread.join(); + } + } finally { + System.clearProperty("sentry.auth.field"); + server.shutdown(); + } + } + + @Test + public void testSuperUser() throws Exception { + final String collection = "testSuperUser"; + setupCollectionWithDocSecurity(collection, 1); + CloudSolrServer server = getCloudSolrServer(collection); + int docCount = 100; + + try { + DocLevelGenerator generator = new DocLevelGenerator(collection, AUTH_FIELD); + generator.generateDocs(server, docCount, "junit_role", "admin_role", 2); + + setAuthenticationUser("solr"); + String [] ids = new String[docCount]; + for (int i = 0; i < docCount; ++i) { + ids[ i ] = Integer.toString(i); + } + QueryResponse response = getIdResponse(new ExpectedResult(server, ids, docCount)); + assertEquals("Wrong number of documents", docCount, response.getResults().getNumFound()); + } finally { + server.shutdown(); + } + } + + private class AuthFieldModifyThread extends Thread { + private CloudSolrServer server; + private SolrInputDocument doc; + private String authField; + private String authFieldValue0; + private String authFieldValue1; + private volatile boolean finished = false; + + private AuthFieldModifyThread(CloudSolrServer server, + SolrInputDocument doc, String authField, + String authFieldValue0, String authFieldValue1) { + this.server = server; + this.doc = doc; + this.authField = authField; + this.authFieldValue0 = authFieldValue0; + this.authFieldValue1 = authFieldValue1; + } + + @Override + public void run() { + while (!finished) { + if (rand.nextBoolean()) { + doc.setField(authField, authFieldValue0); + } else { + doc.setField(authField, authFieldValue1); + } + try { + server.add(doc); + } catch (SolrServerException sse) { + throw new RuntimeException(sse); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + } + + public void setFinished() { + finished = true; + } + } + + private static class ExpectedResult { + public final CloudSolrServer server; + public final String [] ids; + public final int expectedDocs; + public final String fl; + + public ExpectedResult(CloudSolrServer server, String [] ids, int expectedDocs) { + this(server, ids, expectedDocs, null); + } + + public ExpectedResult(CloudSolrServer server, String [] ids, int expectedDocs, String fl) { + this.server = server; + this.ids = ids; + this.expectedDocs = expectedDocs; + this.fl = fl; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/59bbfdc7/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml b/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml index 66449ff..c8bc32f 100644 --- a/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml +++ b/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml @@ -216,6 +216,7 @@ <dynamicField name="*_c" type="currency" indexed="true" stored="true"/> <dynamicField name="*_auth" type="string" indexed="true" stored="false" multiValued="true"/> + <dynamicField name="*_auth_stored" type="string" indexed="true" stored="true" multiValued="true"/> <dynamicField name="ignored_*" type="ignored" multiValued="true"/> <dynamicField name="attr_*" type="text_general" indexed="true" stored="true" multiValued="true"/> http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/59bbfdc7/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/solrconfig-doclevel.xml ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/solrconfig-doclevel.xml b/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/solrconfig-doclevel.xml index 4459c0d..f07d494 100644 --- a/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/solrconfig-doclevel.xml +++ b/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/solrconfig-doclevel.xml @@ -387,14 +387,14 @@ 'soft' commit which only ensures that changes are visible but does not ensure that data is synced to disk. This is faster and more near-realtime friendly than a hard commit. - --> - <autoSoftCommit> - <maxTime>${solr.autoSoftCommit.maxTime:1000}</maxTime> + --> + <autoSoftCommit> + <maxTime>${solr.autoSoftCommit.maxTime:20000}</maxTime> </autoSoftCommit> - + <!-- Update Related Event Listeners - + Various IndexWriter related events can trigger Listeners to take actions. @@ -899,7 +899,7 @@ <!-- realtime get handler, guaranteed to return the latest stored fields of any document, without the need to commit or open a new searcher. The current implementation relies on the updateLog feature being enabled. --> - <requestHandler name="/get" class="solr.RealTimeGetHandler"> + <requestHandler name="/get" class="solr.SecureRealTimeGetHandler"> <lst name="defaults"> <str name="omitHeader">true</str> <str name="wt">json</str> @@ -1351,14 +1351,17 @@ <bool name="enabled">true</bool> <!-- Field where the auth tokens are stored in the document --> - <str name="sentryAuthField">sentry_auth</str> + <str name="sentryAuthField">${sentry.auth.field:sentry_auth}</str> <!-- Auth token defined to allow any role to access the document. Uncomment to enable. --> <str name="allRolesToken">OR</str> </searchComponent> - <!-- A request handler for demonstrating the spellcheck component. + <searchComponent name="secureGet" class="org.apache.solr.handler.component.SecureRealTimeGetComponent" > + </searchComponent> + + <!-- A request handler for demonstrating the spellcheck component. NOTE: This is purely as an example. The whole purpose of the SpellCheckComponent is to hook it into the request handler that http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/59bbfdc7/sentry-tests/sentry-tests-solr/src/test/resources/solr/sentry/test-authz-provider.ini ---------------------------------------------------------------------- diff --git a/sentry-tests/sentry-tests-solr/src/test/resources/solr/sentry/test-authz-provider.ini b/sentry-tests/sentry-tests-solr/src/test/resources/solr/sentry/test-authz-provider.ini index bccc63e..a376cb8 100644 --- a/sentry-tests/sentry-tests-solr/src/test/resources/solr/sentry/test-authz-provider.ini +++ b/sentry-tests/sentry-tests-solr/src/test/resources/solr/sentry/test-authz-provider.ini @@ -31,7 +31,7 @@ admin_all_group = admin_all_role [roles] junit_role = collection=admin, collection=collection1, collection=docLevelCollection, collection=allRolesCollection, collection=testUpdateDeleteOperations docLevel_role = collection=docLevelCollection, collection=testUpdateDeleteOperations -admin_role = collection=admin, collection=collection1, collection=sentryCollection, collection=sentryCollection_underlying1, collection=sentryCollection_underlying2, collection=docLevelCollection, collection=allRolesCollection, collection=testInvariantCollection, collection=testUpdateDeleteOperations, collection=testIndexlevelDoclevelOperations, collection=testUpdateDistribPhase +admin_role = collection=admin, collection=collection1, collection=sentryCollection, collection=sentryCollection_underlying1, collection=sentryCollection_underlying2, collection=docLevelCollection, collection=allRolesCollection, collection=testInvariantCollection, collection=testUpdateDeleteOperations, collection=testIndexlevelDoclevelOperations, collection=testUpdateDistribPhase, collection=testIdvsIds, collection=testIdvsIdsControl, collection=testFl, collection=testNonCommitted, collection=testConcurrentChanges, collection=testSuperUser sentryCollection_query_role = collection=sentryCollection->action=query sentryCollection_update_role = collection=sentryCollection->action=update sentryCollection_query_update_role = collection=sentryCollection->action=query, collection=sentryCollection->action=update
