Repository: incubator-blur Updated Branches: refs/heads/master b736c0a45 -> 9ba62e6c9
Adding read mask to blur document security. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/9ba62e6c Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/9ba62e6c Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/9ba62e6c Branch: refs/heads/master Commit: 9ba62e6c91f4325774b6b40d0e1d8cc5f02306c4 Parents: b736c0a Author: Aaron McCurry <amccu...@gmail.com> Authored: Tue Oct 6 09:39:09 2015 -0400 Committer: Aaron McCurry <amccu...@gmail.com> Committed: Tue Oct 6 09:39:09 2015 -0400 ---------------------------------------------------------------------- .../document/DocumentVisiblityField.java | 3 +- .../security/index/AccessControlFactory.java | 4 + .../security/index/AccessControlReader.java | 4 +- .../security/index/AccessControlWriter.java | 27 ++- .../index/FilterAccessControlFactory.java | 133 ++++++++++++++ .../blur/lucene/security/index/ReadType.java | 2 +- .../security/index/SecureAtomicReader.java | 180 ++++++++++++++++++- .../index/FilterSecureAtomicReaderTest.java | 9 + .../index/SecureAtomicReaderTestBase.java | 123 +++++++++---- 9 files changed, 438 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/main/java/org/apache/blur/lucene/security/document/DocumentVisiblityField.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/document/DocumentVisiblityField.java b/blur-document-security/src/main/java/org/apache/blur/lucene/security/document/DocumentVisiblityField.java index d3fd4f8..09a098f 100644 --- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/document/DocumentVisiblityField.java +++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/document/DocumentVisiblityField.java @@ -25,9 +25,8 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.IndexableField; -public class DocumentVisiblityField extends Field implements IndexableField { +public class DocumentVisiblityField extends Field { public static final FieldType TYPE_NOT_STORED = new FieldType(); public static final FieldType TYPE_STORED = new FieldType(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlFactory.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlFactory.java b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlFactory.java index 25ad35f..40dd486 100644 --- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlFactory.java +++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlFactory.java @@ -24,6 +24,10 @@ public abstract class AccessControlFactory { public abstract String getDiscoverFieldName(); public abstract String getReadFieldName(); + + public abstract String getReadMaskFieldName(); + + public abstract String getReadMaskFieldSuffix(); public abstract AccessControlWriter getWriter(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlReader.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlReader.java b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlReader.java index 51b554a..8ea214d 100644 --- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlReader.java +++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlReader.java @@ -25,11 +25,13 @@ public abstract class AccessControlReader implements Cloneable { public final boolean hasAccess(ReadType type, int docID) throws IOException { switch (type) { - case DOCS_ENUM: + case LIVEDOCS: return readOrDiscoverAccess(docID); case DOCUMENT_FETCH_DISCOVER: return discoverAccess(docID); + case DOCS_ENUM: + case TERMS_ENUM: case BINARY_DOC_VALUE: case DOCUMENT_FETCH_READ: case NORM_VALUE: http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlWriter.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlWriter.java b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlWriter.java index 035cde0..e351745 100644 --- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlWriter.java +++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlWriter.java @@ -24,16 +24,38 @@ import org.apache.lucene.index.IndexableField; public abstract class AccessControlWriter { + /** + * Adds document read visibility rule to document. + */ public abstract Iterable<IndexableField> addReadVisiblity(String read, Iterable<IndexableField> fields); + /** + * Adds document discover visibility rule to document. + */ public abstract Iterable<IndexableField> addDiscoverVisiblity(String discover, Iterable<IndexableField> fields); + /** + * Adds a read mask to document. If a field has been masked the value can not + * be viewed, but if a search utilizes the tokens from the field the document + * can be found. + */ + public abstract Iterable<IndexableField> addReadMask(String fieldToMask, Iterable<IndexableField> fields); + + /** + * This method should be called as the document is being added to the index + * writer. + * + * @param fields + * @return + */ + public abstract Iterable<IndexableField> lastStepBeforeIndexing(Iterable<IndexableField> fields); + protected Iterable<IndexableField> addField(Iterable<IndexableField> fields, IndexableField... fieldsToAdd) { if (fields instanceof Document) { Document document = (Document) fields; if (fieldsToAdd != null) { for (IndexableField field : fieldsToAdd) { - document.add(field); + document.add(field); } } return document; @@ -44,9 +66,10 @@ public abstract class AccessControlWriter { } if (fieldsToAdd != null) { for (IndexableField field : fieldsToAdd) { - list.add(field); + list.add(field); } } return list; } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java index 038b0c7..3b78cf3 100644 --- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java +++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java @@ -17,7 +17,11 @@ package org.apache.blur.lucene.security.index; import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; +import java.util.List; import java.util.Set; import org.apache.blur.lucene.security.DocumentAuthorizations; @@ -25,7 +29,19 @@ import org.apache.blur.lucene.security.document.DocumentVisiblityField; import org.apache.blur.lucene.security.search.BitSetDocumentVisibilityFilterCacheStrategy; import org.apache.blur.lucene.security.search.DocumentVisibilityFilter; import org.apache.blur.lucene.security.search.DocumentVisibilityFilterCacheStrategy; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.document.DoubleField; import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.FloatField; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.LongField; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexableField; @@ -38,6 +54,8 @@ public class FilterAccessControlFactory extends AccessControlFactory { public static final String DISCOVER_FIELD = "_discover_"; public static final String READ_FIELD = "_read_"; + public static final String READ_MASK_FIELD = "_readmask_"; + public static final String READ_MASK_SUFFIX = "$" + READ_MASK_FIELD; @Override public String getDiscoverFieldName() { @@ -50,6 +68,16 @@ public class FilterAccessControlFactory extends AccessControlFactory { } @Override + public String getReadMaskFieldName() { + return READ_MASK_FIELD; + } + + @Override + public String getReadMaskFieldSuffix() { + return READ_MASK_SUFFIX; + } + + @Override public AccessControlWriter getWriter() { return new FilterAccessControlWriter(); } @@ -283,6 +311,111 @@ public class FilterAccessControlFactory extends AccessControlFactory { return addField(fields, new DocumentVisiblityField(DISCOVER_FIELD, discover, Store.YES)); } + @Override + public Iterable<IndexableField> addReadMask(String fieldToMask, Iterable<IndexableField> fields) { + return addField(fields, new StoredField(READ_MASK_FIELD, fieldToMask)); + } + + @Override + public Iterable<IndexableField> lastStepBeforeIndexing(Iterable<IndexableField> fields) { + Set<String> fieldsToMask = getFieldsToMask(fields); + if (fieldsToMask.isEmpty()) { + return fields; + } + List<IndexableField> result = new ArrayList<IndexableField>(); + for (IndexableField field : fields) { + if (fieldsToMask.contains(field.name())) { + // If field is a doc value, then don't bother indexing. + if (!isDocValue(field)) { + if (isStoredField(field)) { + // Stored fields are not indexed, and the document fetch check + // handles the mask. + result.add(field); + } else { + IndexableField mask = createMaskField(field); + result.add(field); + result.add(mask); + } + } + } else { + result.add(field); + } + } + return result; + } + + private static Set<String> getFieldsToMask(Iterable<IndexableField> fields) { + Set<String> result = new HashSet<String>(); + for (IndexableField field : fields) { + if (field.name().equals(READ_MASK_FIELD)) { + result.add(field.stringValue()); + } + } + return result; + } + + private static boolean isStoredField(IndexableField field) { + if (field instanceof StoredField) { + return true; + } + return false; + } + + private static IndexableField createMaskField(IndexableField field) { + if (field instanceof DoubleField) { + DoubleField f = (DoubleField) field; + return new DoubleField(field.name() + READ_MASK_SUFFIX, (double) f.numericValue(), f.fieldType()); + } else if (field instanceof FloatField) { + FloatField f = (FloatField) field; + return new FloatField(field.name() + READ_MASK_SUFFIX, (float) f.numericValue(), f.fieldType()); + } else if (field instanceof IntField) { + IntField f = (IntField) field; + return new IntField(field.name() + READ_MASK_SUFFIX, (int) f.numericValue(), f.fieldType()); + } else if (field instanceof LongField) { + LongField f = (LongField) field; + return new LongField(field.name() + READ_MASK_SUFFIX, (long) f.numericValue(), f.fieldType()); + } else if (field instanceof StringField) { + StringField f = (StringField) field; + if (f.fieldType() == StringField.TYPE_NOT_STORED) { + return new StringField(field.name() + READ_MASK_SUFFIX, f.stringValue(), Store.NO); + } else { + return new StringField(field.name() + READ_MASK_SUFFIX, f.stringValue(), Store.YES); + } + } else if (field instanceof StringField) { + TextField f = (TextField) field; + Reader readerValue = f.readerValue(); + if (readerValue != null) { + return new TextField(field.name() + READ_MASK_SUFFIX, readerValue); + } + TokenStream tokenStreamValue = f.tokenStreamValue(); + if (tokenStreamValue != null) { + return new TextField(field.name() + READ_MASK_SUFFIX, tokenStreamValue); + } + Store s; + if (f.fieldType() == StringField.TYPE_NOT_STORED) { + s = Store.NO; + } else { + s = Store.YES; + } + return new TextField(field.name() + READ_MASK_SUFFIX, f.stringValue(), s); + } else { + throw new RuntimeException("Field [" + field + "] with type [" + field.getClass() + "] is not supported."); + } + } + + private static boolean isDocValue(IndexableField field) { + if (field instanceof BinaryDocValuesField) { + return true; + } else if (field instanceof NumericDocValuesField) { + return true; + } else if (field instanceof SortedDocValuesField) { + return true; + } else if (field instanceof SortedSetDocValuesField) { + return true; + } else { + return false; + } + } } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/ReadType.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/ReadType.java b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/ReadType.java index dec08b0..dcfbe2c 100644 --- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/ReadType.java +++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/ReadType.java @@ -17,5 +17,5 @@ package org.apache.blur.lucene.security.index; public enum ReadType { - LIVEDOCS, DOCUMENT_FETCH_READ, DOCUMENT_FETCH_DISCOVER, NUMERIC_DOC_VALUE, BINARY_DOC_VALUE, SORTED_DOC_VALUE, NORM_VALUE, SORTED_SET_DOC_VALUE, DOCS_ENUM, QUERY + LIVEDOCS, DOCUMENT_FETCH_READ, DOCUMENT_FETCH_DISCOVER, NUMERIC_DOC_VALUE, BINARY_DOC_VALUE, SORTED_DOC_VALUE, NORM_VALUE, SORTED_SET_DOC_VALUE, DOCS_ENUM, QUERY, TERMS_ENUM } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java index 9155769..b81727b 100644 --- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java +++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java @@ -18,6 +18,7 @@ package org.apache.blur.lucene.security.index; import java.io.IOException; import java.util.Collection; +import java.util.HashSet; import java.util.Set; import org.apache.lucene.index.AtomicReader; @@ -33,6 +34,7 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.CompiledAutomaton; @@ -108,11 +110,17 @@ public class SecureAtomicReader extends FilterAtomicReader { @Override public void document(int docID, final StoredFieldVisitor visitor) throws IOException { if (_accessControl.hasAccess(ReadType.DOCUMENT_FETCH_READ, docID)) { - in.document(docID, visitor); + GetReadMaskFields getReadMaskFields = new GetReadMaskFields(); + in.document(docID, getReadMaskFields); + Set<String> readMaskFields = getReadMaskFields.getReadMaskFields(); + if (readMaskFields.isEmpty()) { + in.document(docID, visitor); + } else { + in.document(docID, new ReadMaskStoredFieldVisitor(visitor, readMaskFields)); + } return; } if (_accessControl.hasAccess(ReadType.DOCUMENT_FETCH_DISCOVER, docID)) { - // TODO add way to perform code when visitor runs here.... in.document(docID, new StoredFieldVisitor() { @Override public Status needsField(FieldInfo fieldInfo) throws IOException { @@ -158,6 +166,79 @@ public class SecureAtomicReader extends FilterAtomicReader { } } + private static class ReadMaskStoredFieldVisitor extends StoredFieldVisitor { + + private final StoredFieldVisitor _visitor; + private final Set<String> _readMaskFields; + + public ReadMaskStoredFieldVisitor(StoredFieldVisitor visitor, Set<String> readMaskFields) { + _visitor = visitor; + _readMaskFields = readMaskFields; + } + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + if (_readMaskFields.contains(fieldInfo.name)) { + return Status.NO; + } + return Status.YES; + } + + @Override + public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { + _visitor.binaryField(fieldInfo, value); + } + + @Override + public void stringField(FieldInfo fieldInfo, String value) throws IOException { + _visitor.stringField(fieldInfo, value); + } + + @Override + public void intField(FieldInfo fieldInfo, int value) throws IOException { + _visitor.intField(fieldInfo, value); + } + + @Override + public void longField(FieldInfo fieldInfo, long value) throws IOException { + _visitor.longField(fieldInfo, value); + } + + @Override + public void floatField(FieldInfo fieldInfo, float value) throws IOException { + _visitor.floatField(fieldInfo, value); + } + + @Override + public void doubleField(FieldInfo fieldInfo, double value) throws IOException { + _visitor.doubleField(fieldInfo, value); + } + + } + + private static class GetReadMaskFields extends StoredFieldVisitor { + + private Set<String> _fields = new HashSet<String>(); + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + if (fieldInfo.name.equals(FilterAccessControlFactory.READ_MASK_FIELD)) { + return Status.YES; + } + return Status.NO; + } + + @Override + public void stringField(FieldInfo fieldInfo, String value) throws IOException { + _fields.add(value); + } + + Set<String> getReadMaskFields() { + return _fields; + } + + } + @Override public Fields fields() throws IOException { return new SecureFields(in.fields(), _accessControl, maxDoc()); @@ -315,7 +396,40 @@ public class SecureAtomicReader extends FilterAtomicReader { if (terms == null) { return null; } - return new SecureTerms(terms, _accessControlReader, _maxDoc); + Terms readMask = getReadMaskTerms(in, field); + SecureTerms secureTerms = new SecureTerms(terms, _accessControlReader, _maxDoc); + if (readMask == null) { + return secureTerms; + } else { + return new ReadMaskTerms(secureTerms, readMask); + } + } + + private Terms getReadMaskTerms(Fields in, String field) throws IOException { + return in.terms(field + FilterAccessControlFactory.READ_MASK_SUFFIX); + } + + } + + static class ReadMaskTerms extends FilterTerms { + + private final Terms _readMask; + + public ReadMaskTerms(Terms in, Terms readMask) { + super(in); + _readMask = readMask; + } + + @Override + public TermsEnum iterator(TermsEnum reuse) throws IOException { + TermsEnum maskTermsEnum = _readMask.iterator(null); + return new ReadMaskTermsEnum(maskTermsEnum, in.iterator(reuse)); + } + + @Override + public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) throws IOException { + TermsEnum maskTermsEnum = _readMask.intersect(compiled, startTerm); + return new ReadMaskTermsEnum(maskTermsEnum, in.intersect(compiled, startTerm)); } } @@ -342,6 +456,44 @@ public class SecureAtomicReader extends FilterAtomicReader { } } + static class ReadMaskTermsEnum extends FilterTermsEnum { + + private final TermsEnum _maskTermsEnum; + + public ReadMaskTermsEnum(TermsEnum maskTermsEnum, TermsEnum realTermsEnum) { + super(realTermsEnum); + _maskTermsEnum = maskTermsEnum; + } + + @Override + public BytesRef next() throws IOException { + while (true) { + BytesRef ref = in.next(); + if (ref == null) { + return null; + } + if (!_maskTermsEnum.seekExact(ref, true)) { + return ref; + } + if (checkDocs()) { + return ref; + } + } + } + + private boolean checkDocs() throws IOException { + DocsEnum maskDocsEnum = _maskTermsEnum.docs(null, null, DocsEnum.FLAG_NONE); + DocsEnum docsEnum = in.docs(null, null, DocsEnum.FLAG_NONE); + int docId; + while ((docId = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + if (maskDocsEnum.advance(docId) != docId) { + return true; + } + } + return false; + } + } + static class SecureTermsEnum extends FilterTermsEnum { private final int _maxDoc; @@ -354,6 +506,28 @@ public class SecureAtomicReader extends FilterAtomicReader { } @Override + public BytesRef next() throws IOException { + BytesRef t; + while ((t = in.next()) != null) { + if (hasAccess(t)) { + return t; + } + } + return null; + } + + private boolean hasAccess(BytesRef term) throws IOException { + DocsEnum docsEnum = in.docs(null, null, DocsEnum.FLAG_NONE); + int docId; + while ((docId = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + if (_accessControlReader.hasAccess(ReadType.TERMS_ENUM, docId)) { + return true; + } + } + return false; + } + + @Override public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { Bits secureLiveDocs = getSecureLiveDocs(liveDocs, _maxDoc, _accessControlReader); return in.docs(secureLiveDocs, reuse, flags); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/FilterSecureAtomicReaderTest.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/FilterSecureAtomicReaderTest.java b/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/FilterSecureAtomicReaderTest.java index 748d749..75a0476 100644 --- a/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/FilterSecureAtomicReaderTest.java +++ b/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/FilterSecureAtomicReaderTest.java @@ -16,8 +16,12 @@ */ package org.apache.blur.lucene.security.index; +import java.io.IOException; + import org.apache.blur.lucene.security.index.AccessControlFactory; import org.apache.blur.lucene.security.index.FilterAccessControlFactory; +import org.apache.lucene.queryparser.classic.ParseException; +import org.junit.Test; public class FilterSecureAtomicReaderTest extends SecureAtomicReaderTestBase { @@ -27,5 +31,10 @@ public class FilterSecureAtomicReaderTest extends SecureAtomicReaderTestBase { public AccessControlFactory getAccessControlFactory() { return _accessControlFactory; } + + @Test + public void testTermWalk() throws IOException, ParseException { + super.testTermWalk(); + } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/9ba62e6c/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/SecureAtomicReaderTestBase.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/SecureAtomicReaderTestBase.java b/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/SecureAtomicReaderTestBase.java index f87171a..fc40f42 100644 --- a/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/SecureAtomicReaderTestBase.java +++ b/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/SecureAtomicReaderTestBase.java @@ -28,10 +28,7 @@ import java.util.Iterator; import java.util.List; import java.util.Set; -import org.apache.blur.lucene.security.index.AccessControlFactory; -import org.apache.blur.lucene.security.index.AccessControlReader; -import org.apache.blur.lucene.security.index.AccessControlWriter; -import org.apache.blur.lucene.security.index.SecureAtomicReader; +import org.apache.blur.lucene.security.search.SecureIndexSearcher; import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; @@ -44,7 +41,6 @@ import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -56,8 +52,6 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.QueryParser; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; @@ -69,17 +63,23 @@ import org.junit.Test; public abstract class SecureAtomicReaderTestBase { + private Set<String> discoverableFields = new HashSet<String>(Arrays.asList("info")); + private List<String> readAuthorizations = Arrays.asList("r1"); + private List<String> discoverAuthorizations = Arrays.asList("d1"); + public abstract AccessControlFactory getAccessControlFactory(); @Test public void testLiveDocs() throws IOException { SecureAtomicReader secureReader = getSecureReader(); Bits liveDocs = secureReader.getLiveDocs(); - assertEquals(4, liveDocs.length()); + assertEquals(6, liveDocs.length()); assertTrue(liveDocs.get(0)); assertTrue(liveDocs.get(1)); assertTrue(liveDocs.get(2)); assertFalse(liveDocs.get(3)); + assertTrue(liveDocs.get(4)); + assertTrue(liveDocs.get(5)); secureReader.close(); } @@ -232,30 +232,45 @@ public abstract class SecureAtomicReaderTestBase { public void testTermWalk() throws IOException, ParseException { SecureAtomicReader secureReader = getSecureReader(); Fields fields = secureReader.fields(); - for (String field : fields) { - Terms terms = fields.terms(field); - TermsEnum termsEnum = terms.iterator(null); - BytesRef ref; - while ((ref = termsEnum.next()) != null) { - System.out.println(field + " " + ref.utf8ToString()); - DocsEnum docsEnum = termsEnum.docs(null, null); - int doc; - while ((doc = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - System.out.println(field + " " + ref.utf8ToString() + " " + doc); - } - } - } + // for (String field : fields) { + // Terms terms = fields.terms(field); + // TermsEnum termsEnum = terms.iterator(null); + // BytesRef ref; + // while ((ref = termsEnum.next()) != null) { + // System.out.println(field + " " + ref.utf8ToString()); + // DocsEnum docsEnum = termsEnum.docs(null, null); + // int doc; + // while ((doc = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + // System.out.println(field + " " + ref.utf8ToString() + " " + doc); + // } + // } + // } + + assertEquals(0, getTermCount(fields, "termmask")); //read mask + assertEquals(0, getTermCount(fields, "shouldnotsee")); //discover + assertEquals(1, getTermCount(fields, "test")); + secureReader.close(); } + private int getTermCount(Fields fields, String field) throws IOException { + Terms terms = fields.terms(field); + TermsEnum termsEnum = terms.iterator(null); + int count = 0; + while (termsEnum.next() != null) { + count++; + } + return count; + } + @Test public void testQuery() throws IOException, ParseException { - SecureAtomicReader secureReader = getSecureReader(); + SecureIndexSearcher searcher = getSecureIndexSearcher(); QueryParser parser = new QueryParser(Version.LUCENE_43, "nothing", new KeywordAnalyzer()); Query query = parser.parse("test:test"); - IndexSearcher searcher = new IndexSearcher(secureReader); + TopDocs topDocs = searcher.search(query, 10); - assertEquals(3, topDocs.totalHits); + assertEquals(5, topDocs.totalHits); { int doc = topDocs.scoreDocs[0].doc; assertEquals(0, doc); @@ -277,20 +292,42 @@ public abstract class SecureAtomicReaderTestBase { assertEquals("test", document.get("test")); assertEquals("info", document.get("info")); } + { + int doc = topDocs.scoreDocs[3].doc; + assertEquals(4, doc); + Document document = searcher.doc(doc); + assertNull(document.get("test")); + assertEquals("info", document.get("info")); + } + { + int doc = topDocs.scoreDocs[4].doc; + assertEquals(5, doc); + Document document = searcher.doc(doc); + assertEquals("test", document.get("test")); + assertEquals("info", document.get("info")); + } + } - secureReader.close(); + private SecureIndexSearcher getSecureIndexSearcher() throws IOException { + DirectoryReader reader = createReader(); + return new SecureIndexSearcher(reader, getAccessControlFactory(), Arrays.asList("r1"), Arrays.asList("d1"), + discoverableFields); } private SecureAtomicReader getSecureReader() throws IOException { - AtomicReader baseReader = createReader(); - Set<String> dicoverableFields = new HashSet<String>(); - dicoverableFields.add("info"); - AccessControlReader accessControlReader = getAccessControlFactory().getReader(Arrays.asList("r1"), - Arrays.asList("d1"), dicoverableFields); + AtomicReader baseReader = createAtomicReader(); + AccessControlReader accessControlReader = getAccessControlFactory().getReader(readAuthorizations, + discoverAuthorizations, discoverableFields); return new SecureAtomicReader(baseReader, accessControlReader); } - private AtomicReader createReader() throws IOException { + private AtomicReader createAtomicReader() throws IOException { + DirectoryReader reader = createReader(); + List<AtomicReaderContext> leaves = reader.leaves(); + return leaves.get(0).reader(); + } + + private DirectoryReader createReader() throws IOException { IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer()); Directory dir = new RAMDirectory(); IndexWriter writer = new IndexWriter(dir, conf); @@ -299,17 +336,24 @@ public abstract class SecureAtomicReaderTestBase { addDoc(writer, accessControlWriter, "r2", "d1", 1); addDoc(writer, accessControlWriter, "r1", "d2", 2); addDoc(writer, accessControlWriter, "r2", "d2", 3); + addDoc(writer, accessControlWriter, "r1", "d1", 4, "test"); + addDoc(writer, accessControlWriter, "r1", "d1", 5, "termmask"); writer.close(); - DirectoryReader reader = DirectoryReader.open(dir); - List<AtomicReaderContext> leaves = reader.leaves(); - return leaves.get(0).reader(); + return DirectoryReader.open(dir); } - private void addDoc(IndexWriter writer, AccessControlWriter accessControlWriter, String read, String discover, int doc) - throws IOException { - writer.addDocument(accessControlWriter.addDiscoverVisiblity(discover, - accessControlWriter.addReadVisiblity(read, getDoc(doc)))); + private void addDoc(IndexWriter writer, AccessControlWriter accessControlWriter, String read, String discover, + int doc, String... readMaskFields) throws IOException { + Iterable<IndexableField> fields = getDoc(doc); + fields = accessControlWriter.addReadVisiblity(read, fields); + fields = accessControlWriter.addDiscoverVisiblity(discover, fields); + if (readMaskFields != null) { + for (String readMaskField : readMaskFields) { + fields = accessControlWriter.addReadMask(readMaskField, fields); + } + } + writer.addDocument(accessControlWriter.lastStepBeforeIndexing(fields)); } private Iterable<IndexableField> getDoc(int i) { @@ -319,6 +363,9 @@ public abstract class SecureAtomicReaderTestBase { if (i == 3) { document.add(new StringField("shouldnotsee", "shouldnotsee", Store.YES)); } + if (i == 5) { + document.add(new StringField("termmask", "term", Store.YES)); + } document.add(new NumericDocValuesField("number", i)); document.add(new BinaryDocValuesField("bin", new BytesRef(Integer.toString(i).getBytes()))); document.add(new SortedDocValuesField("sorted", new BytesRef(Integer.toString(i).getBytes())));
