Repository: incubator-blur Updated Branches: refs/heads/master fcc88b168 -> a329ec4f5
Adding a default read mask message that can be set in main properties or on each table. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/a329ec4f Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/a329ec4f Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/a329ec4f Branch: refs/heads/master Commit: a329ec4f5470658882f6f97fcc6af9720ff1558f Parents: fcc88b1 Author: Aaron McCurry <amccu...@gmail.com> Authored: Thu Oct 15 08:23:33 2015 -0400 Committer: Aaron McCurry <amccu...@gmail.com> Committed: Thu Oct 15 08:23:33 2015 -0400 ---------------------------------------------------------------------- .../manager/writer/BlurIndexSimpleWriter.java | 16 +- .../blur/server/BlurSecureIndexSearcher.java | 7 +- .../IndexSearcherCloseableSecureBase.java | 6 +- .../server/BlurSecureIndexSearcherTest.java | 4 +- .../security/index/AccessControlFactory.java | 2 +- .../security/index/AccessControlReader.java | 2 + .../index/FilterAccessControlFactory.java | 17 +- .../security/index/SecureAtomicReader.java | 19 +- .../security/index/SecureDirectoryReader.java | 4 +- .../security/search/SecureIndexSearcher.java | 34 +-- .../blur/lucene/security/IndexSearcherTest.java | 2 +- .../apache/blur/lucene/security/LoadTest.java | 4 +- .../index/SecureAtomicReaderTestBase.java | 8 +- .../AclDiscoverFieldTypeDefinitionTest.java | 2 +- .../type/AclReadFieldTypeDefinitionTest.java | 2 +- .../BaseReadMaskFieldTypeDefinitionTest.java | 245 +++++++++++++++++++ .../DefaultReadMaskFieldTypeDefinitionTest.java | 26 ++ ...oDefaultReadMaskFieldTypeDefinitionTest.java | 26 ++ .../type/ReadMaskFieldTypeDefinitionTest.java | 238 ------------------ .../org/apache/blur/utils/BlurConstants.java | 1 + .../src/main/resources/blur-default.properties | 5 +- .../apache/blur/utils/BlurConstantsTest.java | 2 +- 22 files changed, 386 insertions(+), 286 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java index acc8f39..7cc823a 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/BlurIndexSimpleWriter.java @@ -19,6 +19,7 @@ package org.apache.blur.manager.writer; import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION; import static org.apache.blur.utils.BlurConstants.ACL_DISCOVER; import static org.apache.blur.utils.BlurConstants.ACL_READ; +import static org.apache.blur.utils.BlurConstants.BLUR_RECORD_SECURITY_DEFAULT_READMASK_MESSAGE; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WRITER_SORT_FACTOR; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WRITER_SORT_MEMORY; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_QUEUE_MAX_INMEMORY_LENGTH; @@ -144,6 +145,7 @@ public class BlurIndexSimpleWriter extends BlurIndex { private final Timer _bulkIndexingTimer; private final TimerTask _watchForIdleBulkWriters; private final ThriftCache _thriftCache; + private final String _defaultReadMaskMessage; private Thread _optimizeThread; private Thread _writerOpener; @@ -154,6 +156,7 @@ public class BlurIndexSimpleWriter extends BlurIndex { Timer bulkIndexingTimer, ThriftCache thriftCache) throws IOException { super(shardContext, directory, mergeScheduler, searchExecutor, indexCloser, indexImporterTimer, bulkIndexingTimer, thriftCache); + _thriftCache = thriftCache; _commaSplitter = Splitter.on(','); _bulkWriters = new ConcurrentHashMap<String, BlurIndexSimpleWriter.BulkEntry>(); @@ -166,6 +169,8 @@ public class BlurIndexSimpleWriter extends BlurIndex { _fieldManager = _tableContext.getFieldManager(); _discoverableFields = _tableContext.getDiscoverableFields(); _accessControlFactory = _tableContext.getAccessControlFactory(); + _defaultReadMaskMessage = getDefaultReadMaskMessage(_tableContext); + TableDescriptor descriptor = _tableContext.getDescriptor(); Map<String, String> tableProperties = descriptor.getTableProperties(); if (tableProperties != null) { @@ -235,6 +240,15 @@ public class BlurIndexSimpleWriter extends BlurIndex { _bulkIndexingTimer.schedule(_watchForIdleBulkWriters, delay, delay); } + private String getDefaultReadMaskMessage(TableContext tableContext) { + BlurConfiguration blurConfiguration = tableContext.getBlurConfiguration(); + String message = blurConfiguration.get(BLUR_RECORD_SECURITY_DEFAULT_READMASK_MESSAGE); + if (message == null || message.trim().isEmpty()) { + return null; + } + return message.trim(); + } + private DirectoryReader checkForMemoryLeaks(DirectoryReader wrappped, String message) { DirectoryReader directoryReader = MemoryLeakDetector.record(wrappped, message, _tableContext.getTable(), _shardContext.getShard()); @@ -338,7 +352,7 @@ public class BlurIndexSimpleWriter extends BlurIndex { Collection<String> readAuthorizations = toCollection(readStr); Collection<String> discoverAuthorizations = toCollection(discoverStr); return new IndexSearcherCloseableSecureBase(indexReader, _searchThreadPool, _accessControlFactory, - readAuthorizations, discoverAuthorizations, _discoverableFields) { + readAuthorizations, discoverAuthorizations, _discoverableFields, _defaultReadMaskMessage) { private boolean _closed; @Override http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-core/src/main/java/org/apache/blur/server/BlurSecureIndexSearcher.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/BlurSecureIndexSearcher.java b/blur-core/src/main/java/org/apache/blur/server/BlurSecureIndexSearcher.java index 9df0142..1934dc0 100644 --- a/blur-core/src/main/java/org/apache/blur/server/BlurSecureIndexSearcher.java +++ b/blur-core/src/main/java/org/apache/blur/server/BlurSecureIndexSearcher.java @@ -37,9 +37,10 @@ import org.apache.lucene.search.Query; public class BlurSecureIndexSearcher extends SecureIndexSearcher { public BlurSecureIndexSearcher(IndexReader r, ExecutorService executor, AccessControlFactory accessControlFactory, - Collection<String> readAuthorizations, Collection<String> discoverAuthorizations, Set<String> discoverableFields) - throws IOException { - super(r, executor, accessControlFactory, readAuthorizations, discoverAuthorizations, discoverableFields); + Collection<String> readAuthorizations, Collection<String> discoverAuthorizations, Set<String> discoverableFields, + String defaultReadMaskMessage) throws IOException { + super(r, executor, accessControlFactory, readAuthorizations, discoverAuthorizations, discoverableFields, + defaultReadMaskMessage); } /** http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableSecureBase.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableSecureBase.java b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableSecureBase.java index 40951a7..097938e 100644 --- a/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableSecureBase.java +++ b/blur-core/src/main/java/org/apache/blur/server/IndexSearcherCloseableSecureBase.java @@ -45,8 +45,10 @@ public abstract class IndexSearcherCloseableSecureBase extends BlurSecureIndexSe public IndexSearcherCloseableSecureBase(IndexReader r, ExecutorService executor, AccessControlFactory accessControlFactory, Collection<String> readAuthorizations, - Collection<String> discoverAuthorizations, Set<String> discoverableFields) throws IOException { - super(r, executor, accessControlFactory, readAuthorizations, discoverAuthorizations, discoverableFields); + Collection<String> discoverAuthorizations, Set<String> discoverableFields, String defaultReadMaskMessage) + throws IOException { + super(r, executor, accessControlFactory, readAuthorizations, discoverAuthorizations, discoverableFields, + defaultReadMaskMessage); _executor = executor; } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-core/src/test/java/org/apache/blur/server/BlurSecureIndexSearcherTest.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/server/BlurSecureIndexSearcherTest.java b/blur-core/src/test/java/org/apache/blur/server/BlurSecureIndexSearcherTest.java index 89b3f44..ddde816 100644 --- a/blur-core/src/test/java/org/apache/blur/server/BlurSecureIndexSearcherTest.java +++ b/blur-core/src/test/java/org/apache/blur/server/BlurSecureIndexSearcherTest.java @@ -61,7 +61,7 @@ public class BlurSecureIndexSearcherTest { Collection<String> discoverAuthorizations = new ArrayList<String>(); Set<String> discoverableFields = new HashSet<String>(Arrays.asList("rowid")); BlurSecureIndexSearcher blurSecureIndexSearcher = new BlurSecureIndexSearcher(r, null, accessControlFactory, - readAuthorizations, discoverAuthorizations, discoverableFields); + readAuthorizations, discoverAuthorizations, discoverableFields, null); Query wrapFilter; Query query = new TermQuery(new Term("a", "b")); Filter filter = new Filter() { @@ -97,7 +97,7 @@ public class BlurSecureIndexSearcherTest { Collection<String> discoverAuthorizations = new ArrayList<String>(); Set<String> discoverableFields = new HashSet<String>(Arrays.asList("rowid")); BlurSecureIndexSearcher blurSecureIndexSearcher = new BlurSecureIndexSearcher(r, null, accessControlFactory, - readAuthorizations, discoverAuthorizations, discoverableFields); + readAuthorizations, discoverAuthorizations, discoverableFields, null); Query wrapFilter; Query query = new TermQuery(new Term("a", "b")); Filter filter = new Filter() { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlFactory.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlFactory.java b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlFactory.java index 40dd486..138c6a5 100644 --- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlFactory.java +++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlFactory.java @@ -32,5 +32,5 @@ public abstract class AccessControlFactory { public abstract AccessControlWriter getWriter(); public abstract AccessControlReader getReader(Collection<String> readAuthorizations, Collection<String> discoverAuthorizations, - Set<String> discoverableFields); + Set<String> discoverableFields, String defaultReadMaskMessage); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlReader.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlReader.java b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlReader.java index 8ea214d..888b353 100644 --- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlReader.java +++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/AccessControlReader.java @@ -56,4 +56,6 @@ public abstract class AccessControlReader implements Cloneable { public abstract Filter getQueryFilter() throws IOException; + public abstract String getDefaultReadMaskMessage(); + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java index 4db0ce2..b9414da 100644 --- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java +++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/FilterAccessControlFactory.java @@ -88,8 +88,8 @@ public class FilterAccessControlFactory extends AccessControlFactory { @Override public AccessControlReader getReader(Collection<String> readAuthorizations, - Collection<String> discoverAuthorizations, Set<String> discoverableFields) { - return new FilterAccessControlReader(readAuthorizations, discoverAuthorizations, discoverableFields); + Collection<String> discoverAuthorizations, Set<String> discoverableFields, String defaultReadMaskMessage) { + return new FilterAccessControlReader(readAuthorizations, discoverAuthorizations, discoverableFields, defaultReadMaskMessage); } public static class FilterAccessControlReader extends AccessControlReader { @@ -98,6 +98,7 @@ public class FilterAccessControlFactory extends AccessControlFactory { private final DocumentVisibilityFilter _readDocumentVisibilityFilter; private final DocumentVisibilityFilter _discoverDocumentVisibilityFilter; private final DocumentVisibilityFilterCacheStrategy _filterCacheStrategy; + private final String _defaultReadMaskMessage; private Bits _readBits; private Bits _discoverBits; @@ -108,13 +109,14 @@ public class FilterAccessControlFactory extends AccessControlFactory { private boolean _isClone; public FilterAccessControlReader(Collection<String> readAuthorizations, Collection<String> discoverAuthorizations, - Set<String> discoverableFields) { + Set<String> discoverableFields, String defaultReadMaskMessage) { this(readAuthorizations, discoverAuthorizations, discoverableFields, - BitSetDocumentVisibilityFilterCacheStrategy.INSTANCE); + BitSetDocumentVisibilityFilterCacheStrategy.INSTANCE, defaultReadMaskMessage); } public FilterAccessControlReader(Collection<String> readAuthorizations, Collection<String> discoverAuthorizations, - Set<String> discoverableFields, DocumentVisibilityFilterCacheStrategy filterCacheStrategy) { + Set<String> discoverableFields, DocumentVisibilityFilterCacheStrategy filterCacheStrategy, String defaultReadMaskMessage) { + _defaultReadMaskMessage=defaultReadMaskMessage; _filterCacheStrategy = filterCacheStrategy; if (readAuthorizations == null || readAuthorizations.isEmpty()) { @@ -301,6 +303,11 @@ public class FilterAccessControlFactory extends AccessControlFactory { } }; } + + @Override + public String getDefaultReadMaskMessage() { + return _defaultReadMaskMessage; + } } public static class FilterAccessControlWriter extends AccessControlWriter { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java index 2a17edb..c9a2bfb 100644 --- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java +++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureAtomicReader.java @@ -59,10 +59,10 @@ public class SecureAtomicReader extends FilterAtomicReader { private final AtomicReader _original; public static SecureAtomicReader create(AccessControlFactory accessControlFactory, AtomicReader in, - Collection<String> readAuthorizations, Collection<String> discoverAuthorizations, Set<String> discoverableFields) - throws IOException { + Collection<String> readAuthorizations, Collection<String> discoverAuthorizations, Set<String> discoverableFields, + String defaultReadMaskMessage) throws IOException { AccessControlReader accessControlReader = accessControlFactory.getReader(readAuthorizations, - discoverAuthorizations, discoverableFields); + discoverAuthorizations, discoverableFields, defaultReadMaskMessage); return new SecureAtomicReader(in, accessControlReader); } @@ -114,7 +114,7 @@ public class SecureAtomicReader extends FilterAtomicReader { @Override public void document(int docID, final StoredFieldVisitor visitor) throws IOException { if (_accessControl.hasAccess(ReadType.DOCUMENT_FETCH_READ, docID)) { - GetReadMaskFields getReadMaskFields = new GetReadMaskFields(); + GetReadMaskFields getReadMaskFields = new GetReadMaskFields(_accessControl.getDefaultReadMaskMessage()); in.document(docID, getReadMaskFields); Map<String, String> readMaskFields = getReadMaskFields.getReadMaskFields(); if (readMaskFields.isEmpty()) { @@ -243,8 +243,13 @@ public class SecureAtomicReader extends FilterAtomicReader { private static class GetReadMaskFields extends StoredFieldVisitor { - private Map<String, String> _fieldsAndMessages = new HashMap<String, String>(); - private Splitter splitter = Splitter.on('|'); + private final Map<String, String> _fieldsAndMessages = new HashMap<String, String>(); + private final Splitter splitter = Splitter.on('|'); + private final String _defaultReadMask; + + GetReadMaskFields(String defaultReadMask) { + _defaultReadMask = defaultReadMask == null ? "" : defaultReadMask; + } @Override public Status needsField(FieldInfo fieldInfo) throws IOException { @@ -271,7 +276,7 @@ public class SecureAtomicReader extends FilterAtomicReader { if (message != null) { _fieldsAndMessages.put(field, message); } else { - _fieldsAndMessages.put(field, ""); + _fieldsAndMessages.put(field, _defaultReadMask); } } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureDirectoryReader.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureDirectoryReader.java b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureDirectoryReader.java index 55cffbf..a29977c 100644 --- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureDirectoryReader.java +++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/index/SecureDirectoryReader.java @@ -27,10 +27,10 @@ import org.apache.lucene.index.FilterDirectoryReader; public class SecureDirectoryReader extends FilterDirectoryReader { public static SecureDirectoryReader create(AccessControlFactory accessControlFactory, DirectoryReader in, - Collection<String> readAuthorizations, Collection<String> discoverAuthorizations, Set<String> discoverableFields) + Collection<String> readAuthorizations, Collection<String> discoverAuthorizations, Set<String> discoverableFields, String defaultReadMaskMessage) throws IOException { AccessControlReader accessControlReader = accessControlFactory.getReader(readAuthorizations, - discoverAuthorizations, discoverableFields); + discoverAuthorizations, discoverableFields, defaultReadMaskMessage); return new SecureDirectoryReader(in, accessControlReader); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-document-security/src/main/java/org/apache/blur/lucene/security/search/SecureIndexSearcher.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/main/java/org/apache/blur/lucene/security/search/SecureIndexSearcher.java b/blur-document-security/src/main/java/org/apache/blur/lucene/security/search/SecureIndexSearcher.java index 5714278..6d05358 100644 --- a/blur-document-security/src/main/java/org/apache/blur/lucene/security/search/SecureIndexSearcher.java +++ b/blur-document-security/src/main/java/org/apache/blur/lucene/security/search/SecureIndexSearcher.java @@ -50,36 +50,42 @@ public class SecureIndexSearcher extends IndexSearcher { private final Collection<String> _readAuthorizations; private final Collection<String> _discoverAuthorizations; private final Set<String> _discoverableFields; + private final String _defaultReadMaskMessage; private AccessControlReader _accessControlReader; public SecureIndexSearcher(IndexReader r, AccessControlFactory accessControlFactory, - Collection<String> readAuthorizations, Collection<String> discoverAuthorizations, Set<String> discoverableFields) - throws IOException { - this(r, null, accessControlFactory, readAuthorizations, discoverAuthorizations, discoverableFields); + Collection<String> readAuthorizations, Collection<String> discoverAuthorizations, Set<String> discoverableFields, + String defaultReadMaskMessage) throws IOException { + this(r, null, accessControlFactory, readAuthorizations, discoverAuthorizations, discoverableFields, + defaultReadMaskMessage); } public SecureIndexSearcher(IndexReader r, ExecutorService executor, AccessControlFactory accessControlFactory, - Collection<String> readAuthorizations, Collection<String> discoverAuthorizations, Set<String> discoverableFields) - throws IOException { - this(r.getContext(), executor, accessControlFactory, readAuthorizations, discoverAuthorizations, discoverableFields); + Collection<String> readAuthorizations, Collection<String> discoverAuthorizations, Set<String> discoverableFields, + String defaultReadMaskMessage) throws IOException { + this(r.getContext(), executor, accessControlFactory, readAuthorizations, discoverAuthorizations, + discoverableFields, defaultReadMaskMessage); } public SecureIndexSearcher(IndexReaderContext context, AccessControlFactory accessControlFactory, - Collection<String> readAuthorizations, Collection<String> discoverAuthorizations, Set<String> discoverableFields) - throws IOException { - this(context, null, accessControlFactory, readAuthorizations, discoverAuthorizations, discoverableFields); + Collection<String> readAuthorizations, Collection<String> discoverAuthorizations, Set<String> discoverableFields, + String defaultReadMaskMessage) throws IOException { + this(context, null, accessControlFactory, readAuthorizations, discoverAuthorizations, discoverableFields, + defaultReadMaskMessage); } public SecureIndexSearcher(IndexReaderContext context, ExecutorService executor, AccessControlFactory accessControlFactory, Collection<String> readAuthorizations, - Collection<String> discoverAuthorizations, Set<String> discoverableFields) throws IOException { + Collection<String> discoverAuthorizations, Set<String> discoverableFields, String defaultReadMaskMessage) + throws IOException { super(context, executor); _accessControlFactory = accessControlFactory; _readAuthorizations = readAuthorizations; _discoverAuthorizations = discoverAuthorizations; _discoverableFields = discoverableFields; + _defaultReadMaskMessage = defaultReadMaskMessage; _accessControlReader = _accessControlFactory.getReader(readAuthorizations, discoverAuthorizations, - discoverableFields); + discoverableFields, _defaultReadMaskMessage); _secureIndexReader = getSecureIndexReader(context); List<AtomicReaderContext> leaves = _secureIndexReader.leaves(); _leaveMap = new HashMap<Object, AtomicReaderContext>(); @@ -94,17 +100,17 @@ public class SecureIndexSearcher extends IndexSearcher { protected AtomicReader getSecureAtomicReader(AtomicReader atomicReader) throws IOException { return SecureAtomicReader.create(_accessControlFactory, atomicReader, _readAuthorizations, _discoverAuthorizations, - _discoverableFields); + _discoverableFields, _defaultReadMaskMessage); } protected IndexReader getSecureIndexReader(IndexReaderContext context) throws IOException { IndexReader indexReader = context.reader(); if (indexReader instanceof DirectoryReader) { return SecureDirectoryReader.create(_accessControlFactory, (DirectoryReader) indexReader, _readAuthorizations, - _discoverAuthorizations, _discoverableFields); + _discoverAuthorizations, _discoverableFields, _defaultReadMaskMessage); } else if (indexReader instanceof AtomicReader) { return SecureAtomicReader.create(_accessControlFactory, (AtomicReader) indexReader, _readAuthorizations, - _discoverAuthorizations, _discoverableFields); + _discoverAuthorizations, _discoverableFields, _defaultReadMaskMessage); } throw new IOException("IndexReader type [" + indexReader.getClass() + "] not supported."); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-document-security/src/test/java/org/apache/blur/lucene/security/IndexSearcherTest.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/test/java/org/apache/blur/lucene/security/IndexSearcherTest.java b/blur-document-security/src/test/java/org/apache/blur/lucene/security/IndexSearcherTest.java index 601b8dd..2668721 100644 --- a/blur-document-security/src/test/java/org/apache/blur/lucene/security/IndexSearcherTest.java +++ b/blur-document-security/src/test/java/org/apache/blur/lucene/security/IndexSearcherTest.java @@ -129,7 +129,7 @@ public class IndexSearcherTest { List<AtomicReaderContext> leaves = reader.leaves(); assertEquals(leafCount, leaves.size()); SecureIndexSearcher searcher = new SecureIndexSearcher(reader, getAccessControlFactory(), readAuthorizations, - discoverAuthorizations, toSet(discoverableFields)); + discoverAuthorizations, toSet(discoverableFields), null); TopDocs topDocs; Query query = new MatchAllDocsQuery(); { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-document-security/src/test/java/org/apache/blur/lucene/security/LoadTest.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/test/java/org/apache/blur/lucene/security/LoadTest.java b/blur-document-security/src/test/java/org/apache/blur/lucene/security/LoadTest.java index 80d589a..3abdea9 100644 --- a/blur-document-security/src/test/java/org/apache/blur/lucene/security/LoadTest.java +++ b/blur-document-security/src/test/java/org/apache/blur/lucene/security/LoadTest.java @@ -67,10 +67,10 @@ public class LoadTest { IndexSearcher searcher = new IndexSearcher(reader); SecureIndexSearcher secureIndexSearcher1 = new SecureIndexSearcher(reader, accessControlFactory, - Arrays.asList("nothing"), Arrays.asList("nothing"), new HashSet<String>()); + Arrays.asList("nothing"), Arrays.asList("nothing"), new HashSet<String>(), null); SecureIndexSearcher secureIndexSearcher2 = new SecureIndexSearcher(reader, accessControlFactory, - Arrays.asList("r1"), Arrays.asList("nothing"), new HashSet<String>()); + Arrays.asList("r1"), Arrays.asList("nothing"), new HashSet<String>(), null); MatchAllDocsQuery query = new MatchAllDocsQuery(); for (int p = 0; p < 10; p++) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/SecureAtomicReaderTestBase.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/SecureAtomicReaderTestBase.java b/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/SecureAtomicReaderTestBase.java index 68e72f8..375d0e6 100644 --- a/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/SecureAtomicReaderTestBase.java +++ b/blur-document-security/src/test/java/org/apache/blur/lucene/security/index/SecureAtomicReaderTestBase.java @@ -246,8 +246,8 @@ public abstract class SecureAtomicReaderTestBase { // } // } - assertEquals(0, getTermCount(fields, "termmask")); //read mask - assertEquals(0, getTermCount(fields, "shouldnotsee")); //discover + assertEquals(0, getTermCount(fields, "termmask")); // read mask + assertEquals(0, getTermCount(fields, "shouldnotsee")); // discover assertEquals(1, getTermCount(fields, "test")); secureReader.close(); @@ -311,13 +311,13 @@ public abstract class SecureAtomicReaderTestBase { private SecureIndexSearcher getSecureIndexSearcher() throws IOException { DirectoryReader reader = createReader(); return new SecureIndexSearcher(reader, getAccessControlFactory(), Arrays.asList("r1"), Arrays.asList("d1"), - discoverableFields); + discoverableFields, null); } private SecureAtomicReader getSecureReader() throws IOException { AtomicReader baseReader = createAtomicReader(); AccessControlReader accessControlReader = getAccessControlFactory().getReader(readAuthorizations, - discoverAuthorizations, discoverableFields); + discoverAuthorizations, discoverableFields, null); return new SecureAtomicReader(baseReader, accessControlReader); } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-query/src/test/java/org/apache/blur/analysis/type/AclDiscoverFieldTypeDefinitionTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/analysis/type/AclDiscoverFieldTypeDefinitionTest.java b/blur-query/src/test/java/org/apache/blur/analysis/type/AclDiscoverFieldTypeDefinitionTest.java index 48c79ae..4f4846c 100644 --- a/blur-query/src/test/java/org/apache/blur/analysis/type/AclDiscoverFieldTypeDefinitionTest.java +++ b/blur-query/src/test/java/org/apache/blur/analysis/type/AclDiscoverFieldTypeDefinitionTest.java @@ -185,7 +185,7 @@ public class AclDiscoverFieldTypeDefinitionTest { discoverableFields.add("recordid"); discoverableFields.add("family"); IndexSearcher searcher = new SecureIndexSearcher(reader, getAccessControlFactory(), readAuthorizations, - discoverAuthorizations, discoverableFields); + discoverAuthorizations, discoverableFields, null); TopDocs topDocs = searcher.search(query, 10); assertEquals(expected, topDocs.totalHits); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-query/src/test/java/org/apache/blur/analysis/type/AclReadFieldTypeDefinitionTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/analysis/type/AclReadFieldTypeDefinitionTest.java b/blur-query/src/test/java/org/apache/blur/analysis/type/AclReadFieldTypeDefinitionTest.java index 0a54a96..f36b376 100644 --- a/blur-query/src/test/java/org/apache/blur/analysis/type/AclReadFieldTypeDefinitionTest.java +++ b/blur-query/src/test/java/org/apache/blur/analysis/type/AclReadFieldTypeDefinitionTest.java @@ -178,7 +178,7 @@ public class AclReadFieldTypeDefinitionTest { Collection<String> discoverAuthorizations = null; Set<String> discoverableFields = null; IndexSearcher searcher = new SecureIndexSearcher(reader, getAccessControlFactory(), readAuthorizations, - discoverAuthorizations, discoverableFields); + discoverAuthorizations, discoverableFields, null); TopDocs topDocs = searcher.search(query, 10); assertEquals(expected, topDocs.totalHits); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-query/src/test/java/org/apache/blur/analysis/type/BaseReadMaskFieldTypeDefinitionTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/analysis/type/BaseReadMaskFieldTypeDefinitionTest.java b/blur-query/src/test/java/org/apache/blur/analysis/type/BaseReadMaskFieldTypeDefinitionTest.java new file mode 100644 index 0000000..883b362 --- /dev/null +++ b/blur-query/src/test/java/org/apache/blur/analysis/type/BaseReadMaskFieldTypeDefinitionTest.java @@ -0,0 +1,245 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.analysis.type; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import org.apache.blur.analysis.BaseFieldManager; +import org.apache.blur.analysis.FieldTypeDefinition; +import org.apache.blur.analysis.NoStopWordStandardAnalyzer; +import org.apache.blur.lucene.search.SuperParser; +import org.apache.blur.lucene.security.index.AccessControlFactory; +import org.apache.blur.lucene.security.index.FilterAccessControlFactory; +import org.apache.blur.lucene.security.search.SecureIndexSearcher; +import org.apache.blur.thrift.generated.Column; +import org.apache.blur.thrift.generated.Record; +import org.apache.blur.thrift.generated.ScoreType; +import org.apache.blur.utils.BlurConstants; +import org.apache.hadoop.conf.Configuration; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.Version; +import org.junit.Before; +import org.junit.Test; + +public abstract class BaseReadMaskFieldTypeDefinitionTest { + private static final String FAM = "fam"; + private static final String FAM2 = "fam2"; + + private Directory _dir = new RAMDirectory(); + private AccessControlFactory _accessControlFactory = new FilterAccessControlFactory(); + + private BaseFieldManager _fieldManager; + + @Before + public void setup() throws IOException { + _fieldManager = getFieldManager(new NoStopWordStandardAnalyzer()); + setupFieldManager(_fieldManager); + + List<List<Field>> docs = new ArrayList<List<Field>>(); + { + Record record = new Record(); + record.setFamily(FAM); + record.setRecordId("1234"); + record.addToColumns(new Column("string", "value")); + record.addToColumns(new Column("read", "a&b")); + record.addToColumns(new Column("string2", "value should not read")); + record.addToColumns(new Column("mask", "fam.string2|READ_MASK")); + List<Field> fields = _fieldManager.getFields("1234", record); + fields.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO)); + docs.add(debug(fields)); + } + { + Record record = new Record(); + record.setFamily(FAM); + record.setRecordId("5678"); + record.addToColumns(new Column("string", "value")); + record.addToColumns(new Column("read", "a&c")); + record.addToColumns(new Column("mask", "fam.string")); + docs.add(debug(_fieldManager.getFields("1234", record))); + } + + IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, _fieldManager.getAnalyzerForIndex()); + IndexWriter writer = new IndexWriter(_dir, conf); + writer.addDocuments(docs); + writer.close(); + } + + private List<Field> debug(List<Field> fields) { + // System.out.println("----Document"); + // for (Field field : fields) { + // System.out.println(field); + // } + return fields; + } + + @Test + public void test1RowQuery() throws IOException, ParseException { + test(0, true, null); + } + + @Test + public void test1RecordQuery() throws IOException, ParseException { + test(0, false, null); + } + + @Test + public void test2RowQuery() throws IOException, ParseException { + test(1, true, Arrays.asList("a", "b")); + } + + @Test + public void test2RecordQuery() throws IOException, ParseException { + test(1, false, Arrays.asList("a", "b")); + } + + @Test + public void test3RowQuery() throws IOException, ParseException { + test(1, true, Arrays.asList("a", "b", "c")); + } + + @Test + public void test3RecordQuery() throws IOException, ParseException { + test(2, false, Arrays.asList("a", "b", "c")); + } + + @Test + public void test4RowQuery() throws IOException, ParseException { + test(0, true, Arrays.asList("a")); + } + + @Test + public void test4RecordQuery() throws IOException, ParseException { + test(0, false, Arrays.asList("a")); + } + + private AccessControlFactory getAccessControlFactory() { + return _accessControlFactory; + } + + private void setupFieldManager(BaseFieldManager fieldManager) throws IOException { + fieldManager.addColumnDefinition(FAM, "string", null, false, "string", false, false, null); + fieldManager.addColumnDefinition(FAM, "string2", null, false, "string", false, false, null); + fieldManager.addColumnDefinition(FAM, "read", null, false, "acl-read", false, false, null); + fieldManager.addColumnDefinition(FAM, "mask", null, false, "read-mask", false, false, null); + fieldManager.addColumnDefinition(FAM2, "string", null, false, "string", false, false, null); + fieldManager.addColumnDefinition(FAM2, "read", null, false, "acl-read", false, false, null); + } + + protected BaseFieldManager getFieldManager(Analyzer a) throws IOException { + BaseFieldManager fieldManager = new BaseFieldManager(BlurConstants.SUPER, a, new Configuration()) { + @Override + protected boolean tryToStore(FieldTypeDefinition fieldTypeDefinition, String fieldName) { + return true; + } + + @Override + protected void tryToLoad(String fieldName) { + + } + + @Override + protected List<String> getFieldNamesToLoad() throws IOException { + return new ArrayList<String>(); + } + }; + return fieldManager; + } + + private void test(int expected, boolean rowQuery, Collection<String> readAuthorizations) throws IOException, + ParseException { + DirectoryReader reader = DirectoryReader.open(_dir); + SuperParser parser = new SuperParser(Version.LUCENE_43, _fieldManager, rowQuery, null, ScoreType.SUPER, new Term( + BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE)); + + Query query = parser.parse("fam.string:value"); + + Collection<String> discoverAuthorizations = null; + Set<String> discoverableFields = null; + String defaultReadMask = getDefaultReadMask(); + IndexSearcher searcher = new SecureIndexSearcher(reader, getAccessControlFactory(), readAuthorizations, + discoverAuthorizations, discoverableFields, defaultReadMask); + + checkTerms(searcher, "fam.string2"); + + TopDocs topDocs = searcher.search(query, 10); + assertEquals(expected, topDocs.totalHits); + + for (int hit = 0; hit < topDocs.totalHits; hit++) { + int doc = topDocs.scoreDocs[hit].doc; + Document document = searcher.doc(doc); + String recordId = document.get("recordid"); + if (recordId.equals("1234")) { + String s = document.get("fam.string2"); + assertEquals("READ_MASK", s); + } else if (recordId.equals("5678")) { + String s = document.get("fam.string"); + if (defaultReadMask == null) { + assertNull(s); + } else { + assertEquals(defaultReadMask, s); + } + } + } + + reader.close(); + } + + protected abstract String getDefaultReadMask(); + + private void checkTerms(IndexSearcher searcher, String fieldName) throws IOException { + IndexReader reader = searcher.getIndexReader(); + for (AtomicReaderContext context : reader.leaves()) { + AtomicReader atomicReader = context.reader(); + Fields fields = atomicReader.fields(); + Terms terms = fields.terms(fieldName); + TermsEnum iterator = terms.iterator(null); + BytesRef bytesRef = iterator.next(); + if (bytesRef != null) { + System.out.println(bytesRef.utf8ToString()); + fail("There are only restricted terms for this field [" + fieldName + "]"); + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-query/src/test/java/org/apache/blur/analysis/type/DefaultReadMaskFieldTypeDefinitionTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/analysis/type/DefaultReadMaskFieldTypeDefinitionTest.java b/blur-query/src/test/java/org/apache/blur/analysis/type/DefaultReadMaskFieldTypeDefinitionTest.java new file mode 100644 index 0000000..d0251ac --- /dev/null +++ b/blur-query/src/test/java/org/apache/blur/analysis/type/DefaultReadMaskFieldTypeDefinitionTest.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.analysis.type; + +public class DefaultReadMaskFieldTypeDefinitionTest extends BaseReadMaskFieldTypeDefinitionTest { + + @Override + protected String getDefaultReadMask() { + return "READ_MASK_DEFAULT"; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-query/src/test/java/org/apache/blur/analysis/type/NoDefaultReadMaskFieldTypeDefinitionTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/analysis/type/NoDefaultReadMaskFieldTypeDefinitionTest.java b/blur-query/src/test/java/org/apache/blur/analysis/type/NoDefaultReadMaskFieldTypeDefinitionTest.java new file mode 100644 index 0000000..62e54fc --- /dev/null +++ b/blur-query/src/test/java/org/apache/blur/analysis/type/NoDefaultReadMaskFieldTypeDefinitionTest.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.analysis.type; + +public class NoDefaultReadMaskFieldTypeDefinitionTest extends BaseReadMaskFieldTypeDefinitionTest { + + @Override + protected String getDefaultReadMask() { + return null; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-query/src/test/java/org/apache/blur/analysis/type/ReadMaskFieldTypeDefinitionTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/analysis/type/ReadMaskFieldTypeDefinitionTest.java b/blur-query/src/test/java/org/apache/blur/analysis/type/ReadMaskFieldTypeDefinitionTest.java deleted file mode 100644 index 5d69c7d..0000000 --- a/blur-query/src/test/java/org/apache/blur/analysis/type/ReadMaskFieldTypeDefinitionTest.java +++ /dev/null @@ -1,238 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.blur.analysis.type; - -import static org.junit.Assert.*; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Set; - -import org.apache.blur.analysis.BaseFieldManager; -import org.apache.blur.analysis.FieldTypeDefinition; -import org.apache.blur.analysis.NoStopWordStandardAnalyzer; -import org.apache.blur.lucene.search.SuperParser; -import org.apache.blur.lucene.security.index.AccessControlFactory; -import org.apache.blur.lucene.security.index.FilterAccessControlFactory; -import org.apache.blur.lucene.security.search.SecureIndexSearcher; -import org.apache.blur.thrift.generated.Column; -import org.apache.blur.thrift.generated.Record; -import org.apache.blur.thrift.generated.ScoreType; -import org.apache.blur.utils.BlurConstants; -import org.apache.hadoop.conf.Configuration; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.Field.Store; -import org.apache.lucene.index.AtomicReader; -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.Fields; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.Version; -import org.junit.Before; -import org.junit.Test; - -public class ReadMaskFieldTypeDefinitionTest { - private static final String FAM = "fam"; - private static final String FAM2 = "fam2"; - - private Directory _dir = new RAMDirectory(); - private AccessControlFactory _accessControlFactory = new FilterAccessControlFactory(); - - private BaseFieldManager _fieldManager; - - @Before - public void setup() throws IOException { - _fieldManager = getFieldManager(new NoStopWordStandardAnalyzer()); - setupFieldManager(_fieldManager); - - List<List<Field>> docs = new ArrayList<List<Field>>(); - { - Record record = new Record(); - record.setFamily(FAM); - record.setRecordId("1234"); - record.addToColumns(new Column("string", "value")); - record.addToColumns(new Column("read", "a&b")); - record.addToColumns(new Column("string2", "value should not read")); - record.addToColumns(new Column("mask", "fam.string2|READ_MASK")); - List<Field> fields = _fieldManager.getFields("1234", record); - fields.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO)); - docs.add(debug(fields)); - } - { - Record record = new Record(); - record.setFamily(FAM); - record.setRecordId("5678"); - record.addToColumns(new Column("string", "value")); - record.addToColumns(new Column("read", "a&c")); - record.addToColumns(new Column("mask", "fam.string")); - docs.add(debug(_fieldManager.getFields("1234", record))); - } - - IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, _fieldManager.getAnalyzerForIndex()); - IndexWriter writer = new IndexWriter(_dir, conf); - writer.addDocuments(docs); - writer.close(); - } - - private List<Field> debug(List<Field> fields) { - // System.out.println("----Document"); - // for (Field field : fields) { - // System.out.println(field); - // } - return fields; - } - - @Test - public void test1RowQuery() throws IOException, ParseException { - test(0, true, null); - } - - @Test - public void test1RecordQuery() throws IOException, ParseException { - test(0, false, null); - } - - @Test - public void test2RowQuery() throws IOException, ParseException { - test(1, true, Arrays.asList("a", "b")); - } - - @Test - public void test2RecordQuery() throws IOException, ParseException { - test(1, false, Arrays.asList("a", "b")); - } - - @Test - public void test3RowQuery() throws IOException, ParseException { - test(1, true, Arrays.asList("a", "b", "c")); - } - - @Test - public void test3RecordQuery() throws IOException, ParseException { - test(2, false, Arrays.asList("a", "b", "c")); - } - - @Test - public void test4RowQuery() throws IOException, ParseException { - test(0, true, Arrays.asList("a")); - } - - @Test - public void test4RecordQuery() throws IOException, ParseException { - test(0, false, Arrays.asList("a")); - } - - private AccessControlFactory getAccessControlFactory() { - return _accessControlFactory; - } - - private void setupFieldManager(BaseFieldManager fieldManager) throws IOException { - fieldManager.addColumnDefinition(FAM, "string", null, false, "string", false, false, null); - fieldManager.addColumnDefinition(FAM, "string2", null, false, "string", false, false, null); - fieldManager.addColumnDefinition(FAM, "read", null, false, "acl-read", false, false, null); - fieldManager.addColumnDefinition(FAM, "mask", null, false, "read-mask", false, false, null); - fieldManager.addColumnDefinition(FAM2, "string", null, false, "string", false, false, null); - fieldManager.addColumnDefinition(FAM2, "read", null, false, "acl-read", false, false, null); - } - - protected BaseFieldManager getFieldManager(Analyzer a) throws IOException { - BaseFieldManager fieldManager = new BaseFieldManager(BlurConstants.SUPER, a, new Configuration()) { - @Override - protected boolean tryToStore(FieldTypeDefinition fieldTypeDefinition, String fieldName) { - return true; - } - - @Override - protected void tryToLoad(String fieldName) { - - } - - @Override - protected List<String> getFieldNamesToLoad() throws IOException { - return new ArrayList<String>(); - } - }; - return fieldManager; - } - - private void test(int expected, boolean rowQuery, Collection<String> readAuthorizations) throws IOException, - ParseException { - DirectoryReader reader = DirectoryReader.open(_dir); - SuperParser parser = new SuperParser(Version.LUCENE_43, _fieldManager, rowQuery, null, ScoreType.SUPER, new Term( - BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE)); - - Query query = parser.parse("fam.string:value"); - - Collection<String> discoverAuthorizations = null; - Set<String> discoverableFields = null; - IndexSearcher searcher = new SecureIndexSearcher(reader, getAccessControlFactory(), readAuthorizations, - discoverAuthorizations, discoverableFields); - - checkTerms(searcher, "fam.string2"); - - TopDocs topDocs = searcher.search(query, 10); - assertEquals(expected, topDocs.totalHits); - - for (int hit = 0; hit < topDocs.totalHits; hit++) { - int doc = topDocs.scoreDocs[hit].doc; - Document document = searcher.doc(doc); - String recordId = document.get("recordid"); - if (recordId.equals("1234")) { - String s = document.get("fam.string2"); - assertEquals("READ_MASK", s); - } else if (recordId.equals("5678")) { - String s = document.get("fam.string"); - assertNull(s); - } - } - - reader.close(); - } - - private void checkTerms(IndexSearcher searcher, String fieldName) throws IOException { - IndexReader reader = searcher.getIndexReader(); - for (AtomicReaderContext context : reader.leaves()) { - AtomicReader atomicReader = context.reader(); - Fields fields = atomicReader.fields(); - Terms terms = fields.terms(fieldName); - TermsEnum iterator = terms.iterator(null); - BytesRef bytesRef = iterator.next(); - if (bytesRef != null) { - System.out.println(bytesRef.utf8ToString()); - fail("There are only restricted terms for this field [" + fieldName + "]"); - } - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java ---------------------------------------------------------------------- diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java index 6616946..5f89b40 100644 --- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java +++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java @@ -43,6 +43,7 @@ public class BlurConstants { public static final String BLUR_SHARD_QUEUE_MAX_INMEMORY_LENGTH = "blur.shard.queue.max.inmemory.length"; public static final String BLUR_RECORD_SECURITY = "blur.record.security"; + public static final String BLUR_RECORD_SECURITY_DEFAULT_READMASK_MESSAGE = "blur.record.security.default.readmask.message"; public static final String ACL_DISCOVER = "acl-discover"; public static final String ACL_READ = "acl-read"; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-util/src/main/resources/blur-default.properties ---------------------------------------------------------------------- diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties index a45d8e1..d22c603 100644 --- a/blur-util/src/main/resources/blur-default.properties +++ b/blur-util/src/main/resources/blur-default.properties @@ -37,6 +37,9 @@ blur.server.security.filter.class.<order>= # Enables/disables record level security. blur.record.security=false +# Sets the default readmask message for fields that are read masked. +blur.record.security.default.readmask.message= + # The zookeeper session timeout blur.zookeeper.timeout=90000 @@ -93,7 +96,7 @@ blur.shard.bind.address=0.0.0.0 blur.shard.bind.port=40020 # Experimental stream server. Set threads to positive number to enable. -blur.stream.server.threads=0 +blur.stream.server.threads=10 # The number of command driver threads. blur.shard.command.driver.threads=16 http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/a329ec4f/blur-util/src/test/java/org/apache/blur/utils/BlurConstantsTest.java ---------------------------------------------------------------------- diff --git a/blur-util/src/test/java/org/apache/blur/utils/BlurConstantsTest.java b/blur-util/src/test/java/org/apache/blur/utils/BlurConstantsTest.java index 27b9470..25b5967 100644 --- a/blur-util/src/test/java/org/apache/blur/utils/BlurConstantsTest.java +++ b/blur-util/src/test/java/org/apache/blur/utils/BlurConstantsTest.java @@ -53,7 +53,7 @@ public class BlurConstantsTest { "BLUR_COMMAND_LIB_PATH", "BLUR_TMP_PATH", "BLUR_SECURITY_SASL_TYPE", "BLUR_SECUTIRY_SASL_CUSTOM_CLASS", "BLUR_SECURITY_SASL_LDAP_DOMAIN", "BLUR_SECURITY_SASL_LDAP_BASEDN", "BLUR_SECURITY_SASL_LDAP_URL", "BLUR_SERVER_SECURITY_FILTER_CLASS", "BLUR_FILTER_ALIAS", "BLUR_BULK_UPDATE_WORKING_PATH", - "BLUR_BULK_UPDATE_WORKING_PATH_PERMISSION", "HADOOP_CONF")); + "BLUR_BULK_UPDATE_WORKING_PATH_PERMISSION", "HADOOP_CONF", "BLUR_RECORD_SECURITY_DEFAULT_READMASK_MESSAGE")); } @Test
