Repository: incubator-blur Updated Branches: refs/heads/master 016a38656 -> 3665953e0
Adding new test for acl security. Fixing fetch bug related to acl security. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/3665953e Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/3665953e Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/3665953e Branch: refs/heads/master Commit: 3665953e06920d00224bb6fe67b72fc5d30f453b Parents: 016a386 Author: Aaron McCurry <[email protected]> Authored: Tue Jan 20 06:07:08 2015 -0500 Committer: Aaron McCurry <[email protected]> Committed: Tue Jan 20 06:07:08 2015 -0500 ---------------------------------------------------------------------- .../apache/blur/manager/BlurHighlighter.java | 6 +- .../java/org/apache/blur/utils/BlurUtil.java | 39 +++---- .../blur/thrift/BlurClusterTestSecurity.java | 112 ++++++++++++++++++- .../org/apache/blur/index/AtomicReaderUtil.java | 9 ++ 4 files changed, 137 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3665953e/blur-core/src/main/java/org/apache/blur/manager/BlurHighlighter.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/BlurHighlighter.java b/blur-core/src/main/java/org/apache/blur/manager/BlurHighlighter.java index d79759c..070ca26 100644 --- a/blur-core/src/main/java/org/apache/blur/manager/BlurHighlighter.java +++ b/blur-core/src/main/java/org/apache/blur/manager/BlurHighlighter.java @@ -23,7 +23,7 @@ import org.apache.blur.thrift.generated.HighlightOptions; import org.apache.blur.thrift.generated.Selector; import org.apache.blur.utils.HighlightHelper; import org.apache.lucene.document.Document; -import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.index.AtomicReader; import org.apache.lucene.search.Query; import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; @@ -86,10 +86,10 @@ public class BlurHighlighter { return _fieldManager; } - public Document highlight(int docID, Document document, SegmentReader segmentReader) throws IOException { + public Document highlight(int docID, Document document, AtomicReader atomicReader) throws IOException { Document highlight; try { - highlight = HighlightHelper.highlight(docID, document, _highlightQuery, _fieldManager, segmentReader, _preTag, + highlight = HighlightHelper.highlight(docID, document, _highlightQuery, _fieldManager, atomicReader, _preTag, _postTag); } catch (InvalidTokenOffsetsException e) { throw new IOException(e); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3665953e/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java index c98686b..f6f5597 100644 --- a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java +++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java @@ -63,7 +63,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLongArray; import org.apache.blur.BlurConfiguration; -import org.apache.blur.index.AtomicReaderUtil; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.lucene.search.PrimeDocCache; @@ -115,7 +114,6 @@ import org.apache.lucene.index.BaseCompositeReaderUtil; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.SegmentReader; import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; @@ -895,20 +893,19 @@ public class BlurUtil { int readerBase = BaseCompositeReaderUtil.readerBase(indexReader, readerIndex); int primeDocId = notAdjustedPrimeDocId - readerBase; IndexReader orgReader = sequentialSubReaders.get(readerIndex); - SegmentReader sReader = AtomicReaderUtil.getSegmentReader(orgReader); - if (sReader != null) { - SegmentReader segmentReader = (SegmentReader) sReader; - Bits liveDocs = segmentReader.getLiveDocs(); + if (orgReader != null && orgReader instanceof AtomicReader) { + AtomicReader atomicReader = (AtomicReader) orgReader; + Bits liveDocs = atomicReader.getLiveDocs(); - OpenBitSet bitSet = PrimeDocCache.getPrimeDocBitSet(primeDocTerm, segmentReader); + OpenBitSet bitSet = PrimeDocCache.getPrimeDocBitSet(primeDocTerm, atomicReader); int nextPrimeDoc = bitSet.nextSetBit(primeDocId + 1); int numberOfDocsInRow; if (nextPrimeDoc == -1) { - numberOfDocsInRow = segmentReader.maxDoc() - primeDocId; + numberOfDocsInRow = atomicReader.maxDoc() - primeDocId; } else { numberOfDocsInRow = nextPrimeDoc - primeDocId; } - OpenBitSet docsInRowSpanToFetch = getDocsToFetch(segmentReader, selector, primeDocId, numberOfDocsInRow, + OpenBitSet docsInRowSpanToFetch = getDocsToFetch(atomicReader, selector, primeDocId, numberOfDocsInRow, liveDocs, filter, totalRecords); int start = selector.getStartRecord(); int maxDocsToFetch = selector.getMaxRecordsToFetch(); @@ -934,10 +931,10 @@ public class BlurUtil { if (docsInRowSpanToFetch.fastGet(cursor)) { maxDocsToFetch--; int docID = primeDocId + cursor; - segmentReader.document(docID, fieldSelector); + atomicReader.document(docID, fieldSelector); Document document = fieldSelector.getDocument(); if (highlighter.shouldHighlight()) { - docs.add(highlighter.highlight(docID, document, segmentReader)); + docs.add(highlighter.highlight(docID, document, atomicReader)); } else { docs.add(document); } @@ -953,7 +950,7 @@ public class BlurUtil { } return orderDocsBasedOnFamilyOrder(docs, selector); } else { - throw new IOException("Expecting a segment reader got a [" + orgReader + "]"); + throw new IOException("Expecting a atomic reader got a [" + orgReader + "]"); } } throw new IOException("IndexReader [" + reader + "] is not a basecompsitereader"); @@ -1009,26 +1006,26 @@ public class BlurUtil { return docStartingPosition; } - private static OpenBitSet getDocsToFetch(SegmentReader segmentReader, Selector selector, int primeDocRowId, + private static OpenBitSet getDocsToFetch(AtomicReader atomicReader, Selector selector, int primeDocRowId, int numberOfDocsInRow, Bits liveDocs, Filter filter, AtomicInteger totalRecords) throws IOException { Set<String> alreadyProcessed = new HashSet<String>(); OpenBitSet bits = new OpenBitSet(numberOfDocsInRow); OpenBitSet mask = null; if (filter != null) { - DocIdSet docIdSet = filter.getDocIdSet(segmentReader.getContext(), liveDocs); + DocIdSet docIdSet = filter.getDocIdSet(atomicReader.getContext(), liveDocs); mask = getMask(docIdSet, primeDocRowId, numberOfDocsInRow); } Set<String> columnFamiliesToFetch = selector.getColumnFamiliesToFetch(); boolean fetchAll = true; if (columnFamiliesToFetch != null) { fetchAll = false; - applyFamilies(alreadyProcessed, bits, columnFamiliesToFetch, segmentReader, primeDocRowId, numberOfDocsInRow, + applyFamilies(alreadyProcessed, bits, columnFamiliesToFetch, atomicReader, primeDocRowId, numberOfDocsInRow, liveDocs); } Map<String, Set<String>> columnsToFetch = selector.getColumnsToFetch(); if (columnsToFetch != null) { fetchAll = false; - applyColumns(alreadyProcessed, bits, columnsToFetch, segmentReader, primeDocRowId, numberOfDocsInRow, liveDocs); + applyColumns(alreadyProcessed, bits, columnsToFetch, atomicReader, primeDocRowId, numberOfDocsInRow, liveDocs); } if (fetchAll) { bits.set(0, numberOfDocsInRow); @@ -1056,18 +1053,18 @@ public class BlurUtil { } private static void applyColumns(Set<String> alreadyProcessed, OpenBitSet bits, - Map<String, Set<String>> columnsToFetch, SegmentReader segmentReader, int primeDocRowId, int numberOfDocsInRow, + Map<String, Set<String>> columnsToFetch, AtomicReader atomicReader, int primeDocRowId, int numberOfDocsInRow, Bits liveDocs) throws IOException { for (String family : columnsToFetch.keySet()) { if (!alreadyProcessed.contains(family)) { - applyFamily(bits, family, segmentReader, primeDocRowId, numberOfDocsInRow, liveDocs); + applyFamily(bits, family, atomicReader, primeDocRowId, numberOfDocsInRow, liveDocs); alreadyProcessed.add(family); } } } private static void applyFamilies(Set<String> alreadyProcessed, OpenBitSet bits, Set<String> columnFamiliesToFetch, - SegmentReader segmentReader, int primeDocRowId, int numberOfDocsInRow, Bits liveDocs) throws IOException { + AtomicReader segmentReader, int primeDocRowId, int numberOfDocsInRow, Bits liveDocs) throws IOException { for (String family : columnFamiliesToFetch) { if (!alreadyProcessed.contains(family)) { applyFamily(bits, family, segmentReader, primeDocRowId, numberOfDocsInRow, liveDocs); @@ -1076,9 +1073,9 @@ public class BlurUtil { } } - private static void applyFamily(OpenBitSet bits, String family, SegmentReader segmentReader, int primeDocRowId, + private static void applyFamily(OpenBitSet bits, String family, AtomicReader atomicReader, int primeDocRowId, int numberOfDocsInRow, Bits liveDocs) throws IOException { - Fields fields = segmentReader.fields(); + Fields fields = atomicReader.fields(); Terms terms = fields.terms(BlurConstants.FAMILY); TermsEnum iterator = terms.iterator(null); BytesRef text = new BytesRef(family); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3665953e/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestSecurity.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestSecurity.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestSecurity.java index b669406..f3e01bf 100644 --- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestSecurity.java +++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestSecurity.java @@ -16,24 +16,47 @@ ing ok * Licensed to the Apache Software Foundation (ASF) under one or more */ package org.apache.blur.thrift; +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.UUID; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; +import org.apache.blur.thrift.generated.BlurQuery; +import org.apache.blur.thrift.generated.BlurResult; +import org.apache.blur.thrift.generated.BlurResults; import org.apache.blur.thrift.generated.Column; import org.apache.blur.thrift.generated.ColumnDefinition; +import org.apache.blur.thrift.generated.FetchResult; +import org.apache.blur.thrift.generated.FetchRowResult; +import org.apache.blur.thrift.generated.Query; import org.apache.blur.thrift.generated.Record; import org.apache.blur.thrift.generated.RecordMutation; +import org.apache.blur.thrift.generated.Row; import org.apache.blur.thrift.generated.RowMutation; +import org.apache.blur.thrift.generated.RowMutationType; +import org.apache.blur.thrift.generated.Selector; import org.apache.blur.thrift.generated.TableDescriptor; +import org.apache.blur.thrift.util.BlurThriftHelper; import org.apache.blur.user.User; +import org.apache.blur.user.UserContext; import org.apache.blur.utils.BlurConstants; +import org.junit.Test; public class BlurClusterTestSecurity extends BlurClusterTestBase { + private static final String ACL_DISCOVER = "acl-discover"; + private static final String TEST = "test"; + private static final String ACL_READ = "acl-read"; private static final String DISCOVER = "discover"; private static final String READ = "read"; @@ -47,8 +70,8 @@ public class BlurClusterTestSecurity extends BlurClusterTestBase { List<RecordMutation> mutations = rowMutation.getRecordMutations(); for (RecordMutation mutation : mutations) { Record record = mutation.getRecord(); - record.addToColumns(new Column("acl-read", READ)); - record.addToColumns(new Column("acl-discover", DISCOVER)); + record.addToColumns(new Column(ACL_READ, READ)); + record.addToColumns(new Column(ACL_DISCOVER, DISCOVER)); } return rowMutation; } @@ -57,9 +80,9 @@ public class BlurClusterTestSecurity extends BlurClusterTestBase { protected void postTableCreate(TableDescriptor tableDescriptor, Iface client) { String name = tableDescriptor.getName(); try { - client.addColumnDefinition(name, new ColumnDefinition("test", "acl-read", null, false, "acl-read", null, false)); - client.addColumnDefinition(name, new ColumnDefinition("test", "acl-discover", null, false, "acl-discover", null, - false)); + client.addColumnDefinition(name, new ColumnDefinition(TEST, ACL_READ, null, false, ACL_READ, null, false)); + client + .addColumnDefinition(name, new ColumnDefinition(TEST, ACL_DISCOVER, null, false, ACL_DISCOVER, null, false)); } catch (BlurException e) { throw new RuntimeException(e); } catch (TException e) { @@ -80,4 +103,83 @@ public class BlurClusterTestSecurity extends BlurClusterTestBase { return attributes; } + @Test + public void testSecurityReadOnlyRowQuery() throws BlurException, TException, IOException { + String tableName = "testSecurity1"; + createTable(tableName); + Iface client = getClient(); + String bulkId = UUID.randomUUID().toString(); + client.bulkMutateStart(bulkId); + for (int i = 0; i < 1000; i++) { + RowMutation mutation1 = new RowMutation(tableName, getRowId(), RowMutationType.REPLACE_ROW, getRecordMutations( + "a", "a&b", "(a&b)|c")); + RowMutation mutation2 = new RowMutation(tableName, getRowId(), RowMutationType.REPLACE_ROW, getRecordMutations( + "b", "b&c", "a|(b&c)")); + RowMutation mutation3 = new RowMutation(tableName, getRowId(), RowMutationType.REPLACE_ROW, getRecordMutations( + "c", "c&a", "(a&c)|b")); + client.bulkMutateAddMultiple(bulkId, Arrays.asList(mutation1, mutation2, mutation3)); + } + client.bulkMutateFinish(bulkId, true, true); + + Map<String, String> attributes = new HashMap<String, String>(); + attributes.put(BlurConstants.ACL_READ, "a"); + User user = new User("testuser", attributes); + + UserContext.setUser(user); + BlurQuery blurQuery = new BlurQuery(); + Query query = new Query(); + query.setRowQuery(true); + query.setQuery("test.test:value"); + blurQuery.setQuery(query); + blurQuery.setSelector(new Selector()); + BlurResults results = client.query(tableName, blurQuery); + assertEquals(2000, results.getTotalResults()); + + Set<String> aclChecks = new HashSet<String>(); + aclChecks.add("a"); + aclChecks.add("a|(b&c)"); + + for (BlurResult result : results.getResults()) { + FetchResult fetchResult = result.getFetchResult(); + assertNotNull(fetchResult); + FetchRowResult rowResult = fetchResult.getRowResult(); + assertNotNull(rowResult); + Row row = rowResult.getRow(); + assertNotNull(row); + List<Record> records = row.getRecords(); + assertEquals(1, records.size()); + for (Record record : records) { + Column column = findColumn(record, ACL_READ); + String value = column.getValue(); + assertTrue(aclChecks.contains(value)); + } + } + + UserContext.reset(); + } + + private Column findColumn(Record record, String name) { + for (Column column : record.getColumns()) { + if (column.getName().equals(name)) { + return column; + } + } + return null; + } + + private List<RecordMutation> getRecordMutations(String... readAcls) { + List<RecordMutation> recordMutations = new ArrayList<RecordMutation>(); + for (int i = 0; i < readAcls.length; i++) { + String recordId = Integer.toString(i); + RecordMutation recordMutation = BlurThriftHelper.newRecordMutation(TEST, recordId, + BlurThriftHelper.newColumn(TEST, "value"), BlurThriftHelper.newColumn(ACL_READ, readAcls[i])); + recordMutations.add(recordMutation); + } + return recordMutations; + } + + private String getRowId() { + return UUID.randomUUID().toString(); + } + } http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/3665953e/blur-store/src/main/java/org/apache/blur/index/AtomicReaderUtil.java ---------------------------------------------------------------------- diff --git a/blur-store/src/main/java/org/apache/blur/index/AtomicReaderUtil.java b/blur-store/src/main/java/org/apache/blur/index/AtomicReaderUtil.java index e10403e..6e12580 100644 --- a/blur-store/src/main/java/org/apache/blur/index/AtomicReaderUtil.java +++ b/blur-store/src/main/java/org/apache/blur/index/AtomicReaderUtil.java @@ -26,6 +26,15 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.SegmentReader; public class AtomicReaderUtil { + + /** + * WARNING!!! This method can bypass access control. Use only when you are + * sure you know what you are doing! + * + * @param indexReader + * @return + * @throws IOException + */ public static SegmentReader getSegmentReader(IndexReader indexReader) throws IOException { if (indexReader instanceof SegmentReader) { return (SegmentReader) indexReader;
