Repository: incubator-blur Updated Branches: refs/heads/master 65640200a -> 991fb0435
First patch of updates. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/0e8d0e82 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/0e8d0e82 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/0e8d0e82 Branch: refs/heads/master Commit: 0e8d0e82700e2a056ab4bab72ce80bab29946d6b Parents: 6564020 Author: Aaron McCurry <amccu...@gmail.com> Authored: Sat May 7 13:09:44 2016 -0400 Committer: Aaron McCurry <amccu...@gmail.com> Committed: Sat May 7 13:09:44 2016 -0400 ---------------------------------------------------------------------- .../apache/blur/command/CommandStatusUtil.java | 83 +++++++++++++ .../manager/writer/MergeSortRowIdLookup.java | 104 ++++++++++++++++ ...cumentVisibilityFilterCacheStrategyTest.java | 122 +++++++++++++++++++ 3 files changed, 309 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e8d0e82/blur-core/src/main/java/org/apache/blur/command/CommandStatusUtil.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/command/CommandStatusUtil.java b/blur-core/src/main/java/org/apache/blur/command/CommandStatusUtil.java new file mode 100644 index 0000000..eeb9d08 --- /dev/null +++ b/blur-core/src/main/java/org/apache/blur/command/CommandStatusUtil.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.command; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.Map.Entry; + +import org.apache.blur.thrift.generated.CommandStatus; +import org.apache.blur.thrift.generated.CommandStatusState; + +public class CommandStatusUtil { + + public static CommandStatus mergeCommandStatus(CommandStatus cs1, CommandStatus cs2) { + if (cs1 == null && cs2 == null) { + return null; + } else if (cs1 == null) { + return cs2; + } else if (cs2 == null) { + return cs1; + } else { + Map<String, Map<CommandStatusState, Long>> serverStateMap1 = cs1.getServerStateMap(); + Map<String, Map<CommandStatusState, Long>> serverStateMap2 = cs2.getServerStateMap(); + Map<String, Map<CommandStatusState, Long>> merge = mergeServerStateMap(serverStateMap1, serverStateMap2); + return new CommandStatus(cs1.getExecutionId(), cs1.getCommandName(), cs1.getArguments(), merge, cs1.getUser()); + } + } + + private static Map<String, Map<CommandStatusState, Long>> mergeServerStateMap( + Map<String, Map<CommandStatusState, Long>> serverStateMap1, + Map<String, Map<CommandStatusState, Long>> serverStateMap2) { + Map<String, Map<CommandStatusState, Long>> result = new HashMap<String, Map<CommandStatusState, Long>>(); + Set<String> keys = new HashSet<String>(); + keys.addAll(serverStateMap1.keySet()); + keys.addAll(serverStateMap2.keySet()); + for (String key : keys) { + Map<CommandStatusState, Long> css1 = serverStateMap1.get(key); + Map<CommandStatusState, Long> css2 = serverStateMap2.get(key); + result.put(key, mergeCommandStatusState(css1, css2)); + } + return result; + } + + private static Map<CommandStatusState, Long> mergeCommandStatusState(Map<CommandStatusState, Long> css1, + Map<CommandStatusState, Long> css2) { + if (css1 == null && css2 == null) { + return new HashMap<CommandStatusState, Long>(); + } else if (css1 == null) { + return css2; + } else if (css2 == null) { + return css1; + } else { + Map<CommandStatusState, Long> result = new HashMap<CommandStatusState, Long>(css1); + for (Entry<CommandStatusState, Long> e : css2.entrySet()) { + CommandStatusState key = e.getKey(); + Long l = result.get(key); + Long value = e.getValue(); + if (l == null) { + result.put(key, value); + } else { + result.put(key, l + value); + } + } + return result; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e8d0e82/blur-core/src/main/java/org/apache/blur/manager/writer/MergeSortRowIdLookup.java ---------------------------------------------------------------------- diff --git a/blur-core/src/main/java/org/apache/blur/manager/writer/MergeSortRowIdLookup.java b/blur-core/src/main/java/org/apache/blur/manager/writer/MergeSortRowIdLookup.java new file mode 100644 index 0000000..b500a4f --- /dev/null +++ b/blur-core/src/main/java/org/apache/blur/manager/writer/MergeSortRowIdLookup.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.manager.writer; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.blur.utils.BlurConstants; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; + +public class MergeSortRowIdLookup { + + public interface Action { + void found(AtomicReader reader, Bits liveDocs, TermsEnum termsEnum) throws IOException; + } + + private final List<TermsEnumReader> _termsEnumList = new ArrayList<TermsEnumReader>(); + + public MergeSortRowIdLookup(IndexReader indexReader) throws IOException { + if (indexReader instanceof AtomicReader) { + addAtomicReader((AtomicReader) indexReader); + } else { + for (AtomicReaderContext context : indexReader.leaves()) { + addAtomicReader(context.reader()); + } + } + } + + private void addAtomicReader(AtomicReader atomicReader) throws IOException { + Terms terms = atomicReader.fields().terms(BlurConstants.ROW_ID); + TermsEnum termsEnum = terms.iterator(null); + _termsEnumList.add(new TermsEnumReader(termsEnum, atomicReader)); + } + + public void lookup(BytesRef rowId, Action action) throws IOException { + advance(_termsEnumList, rowId); + sort(_termsEnumList); + for (TermsEnumReader reader : _termsEnumList) { + if (reader._termsEnum.term().equals(rowId)) { + action.found(reader._reader, reader._liveDocs, reader._termsEnum); + } + } + } + + private static void advance(List<TermsEnumReader> termsEnumList, BytesRef rowId) throws IOException { + for (TermsEnumReader reader : termsEnumList) { + BytesRef term = reader._termsEnum.term(); + if (term.compareTo(rowId) < 0) { + reader._termsEnum.seekCeil(rowId); + } + } + } + + private static void sort(List<TermsEnumReader> termsEnumList) { + Collections.sort(termsEnumList); + } + + private static class TermsEnumReader implements Comparable<TermsEnumReader> { + + final Bits _liveDocs; + final TermsEnum _termsEnum; + final AtomicReader _reader; + + TermsEnumReader(TermsEnum termsEnum, AtomicReader reader) { + _termsEnum = termsEnum; + _reader = reader; + _liveDocs = reader.getLiveDocs(); + } + + @Override + public int compareTo(TermsEnumReader o) { + try { + BytesRef t1 = _termsEnum.term(); + BytesRef t2 = o._termsEnum.term(); + return t1.compareTo(t2); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/0e8d0e82/blur-document-security/src/test/java/org/apache/blur/lucene/security/search/BitSetDocumentVisibilityFilterCacheStrategyTest.java ---------------------------------------------------------------------- diff --git a/blur-document-security/src/test/java/org/apache/blur/lucene/security/search/BitSetDocumentVisibilityFilterCacheStrategyTest.java b/blur-document-security/src/test/java/org/apache/blur/lucene/security/search/BitSetDocumentVisibilityFilterCacheStrategyTest.java new file mode 100644 index 0000000..be289b0 --- /dev/null +++ b/blur-document-security/src/test/java/org/apache/blur/lucene/security/search/BitSetDocumentVisibilityFilterCacheStrategyTest.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.lucene.security.search; + +import static org.junit.Assert.*; + +import java.io.IOException; + +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.OpenBitSet; +import org.junit.Test; + +public class BitSetDocumentVisibilityFilterCacheStrategyTest { + + @Test + public void testIdFullySet1() { + int maxDoc = 10; + OpenBitSet bitSet = new OpenBitSet(maxDoc); + assertFalse(BitSetDocumentVisibilityFilterCacheStrategy.isFullySet(maxDoc, bitSet, bitSet.cardinality())); + } + + @Test + public void testIdFullySet2() { + int maxDoc = 10; + OpenBitSet bitSet = new OpenBitSet(maxDoc); + for (int d = 0; d < maxDoc; d++) { + bitSet.set(d); + } + assertTrue(BitSetDocumentVisibilityFilterCacheStrategy.isFullySet(maxDoc, bitSet, bitSet.cardinality())); + } + + @Test + public void testIdFullyEmpty1() { + int maxDoc = 10; + OpenBitSet bitSet = new OpenBitSet(maxDoc); + assertTrue(BitSetDocumentVisibilityFilterCacheStrategy.isFullyEmpty(bitSet, bitSet.cardinality())); + } + + @Test + public void testIdFullyEmpty2() { + int maxDoc = 10; + OpenBitSet bitSet = new OpenBitSet(maxDoc); + bitSet.set(3); + assertFalse(BitSetDocumentVisibilityFilterCacheStrategy.isFullyEmpty(bitSet, bitSet.cardinality())); + } + + @Test + public void testFullySetDocIdSet() throws IOException { + int len = 10; + DocIdSet docIdSet = BitSetDocumentVisibilityFilterCacheStrategy.getFullySetDocIdSet(len); + Bits bits = docIdSet.bits(); + assertEquals(len, bits.length()); + for (int i = 0; i < len; i++) { + assertTrue(bits.get(i)); + } + assertTrue(docIdSet.isCacheable()); + { + DocIdSetIterator iterator = docIdSet.iterator(); + int adoc; + int edoc = 0; + assertEquals(-1, iterator.docID()); + while ((adoc = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + assertEquals(edoc, adoc); + assertEquals(edoc, iterator.docID()); + edoc++; + } + assertEquals(len, edoc); + } + { + DocIdSetIterator iterator = docIdSet.iterator(); + int adoc; + int edoc = 0; + assertEquals(-1, iterator.docID()); + while ((adoc = iterator.advance(edoc)) != DocIdSetIterator.NO_MORE_DOCS) { + assertEquals(edoc, adoc); + assertEquals(edoc, iterator.docID()); + edoc++; + } + assertEquals(len, edoc); + } + } + + @Test + public void testFullyEmptyDocIdSet() throws IOException { + int len = 10; + DocIdSet docIdSet = BitSetDocumentVisibilityFilterCacheStrategy.getFullyEmptyDocIdSet(len); + Bits bits = docIdSet.bits(); + assertEquals(len, bits.length()); + for (int i = 0; i < len; i++) { + assertFalse(bits.get(i)); + } + assertTrue(docIdSet.isCacheable()); + { + DocIdSetIterator iterator = docIdSet.iterator(); + assertEquals(-1, iterator.docID()); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.docID()); + } + { + DocIdSetIterator iterator = docIdSet.iterator(); + assertEquals(-1, iterator.docID()); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.advance(0)); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.docID()); + } + } +}
