Author: catholicon
Date: Wed Dec 5 00:21:54 2018
New Revision: 1848182
URL: http://svn.apache.org/viewvc?rev=1848182&view=rev
Log:
OAK-7929: Incorrect Facet Count With Large Dataset and ACLs
* Allow secure and sampleSize under <index-def>/facets
* These can be overridden by JVM command line param
* Inject seed into index definition during indexing cycle
* Rename FilteredFacetCounts to SecureFacetCounts to serve for secure
configuration
* StatisticalSortedSetDocValueFacetCounts would do random sampling of
results wrt what ratio are accessible. Use this ratio to extrapolate
facet counts returned by index
* Add TemporarySystemProperty rule to ease writing tests which set system
property
Added:
jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/junit/TemporarySystemProperty.java
(with props)
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SecureSortedSetDocValuesFacetCounts.java
(with props)
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/StatisticalSortedSetDocValuesFacetCounts.java
(with props)
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/SecureFacetTest.java
(with props)
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinitionFacetConfigTest.java
(with props)
Removed:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FilteredSortedSetDocValuesFacetCounts.java
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FacetHelper.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
jackrabbit/oak/trunk/oak-search/pom.xml
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java
Added:
jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/junit/TemporarySystemProperty.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/junit/TemporarySystemProperty.java?rev=1848182&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/junit/TemporarySystemProperty.java
(added)
+++
jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/junit/TemporarySystemProperty.java
Wed Dec 5 00:21:54 2018
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.commons.junit;
+
+import org.junit.rules.ExternalResource;
+
+import java.util.Properties;
+
+public class TemporarySystemProperty extends ExternalResource {
+
+ Properties oldProps = new Properties();
+
+ @Override
+ protected void before() {
+ oldProps.putAll(System.getProperties());
+ }
+
+ @Override
+ protected void after() {
+ System.setProperties(oldProps);
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/junit/TemporarySystemProperty.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1848182&r1=1848181&r2=1848182&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
Wed Dec 5 00:21:54 2018
@@ -120,7 +120,6 @@ import org.apache.lucene.search.highligh
import org.apache.lucene.search.postingshighlight.PostingsHighlighter;
import org.apache.lucene.search.spell.SuggestWord;
import org.apache.lucene.search.suggest.Lookup;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -129,7 +128,6 @@ import org.slf4j.LoggerFactory;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.base.Predicates.in;
import static com.google.common.base.Predicates.notNull;
import static com.google.common.collect.Lists.newArrayListWithCapacity;
import static org.apache.jackrabbit.JcrConstants.JCR_MIXINTYPES;
@@ -353,7 +351,7 @@ public class LucenePropertyIndex extends
long f = PERF_LOGGER.start();
if (facetProvider == null) {
facetProvider = new LuceneFacetProvider(
- FacetHelper.getFacets(searcher, query,
docs, plan, indexNode.getDefinition().isSecureFacets())
+ FacetHelper.getFacets(searcher, query,
plan, indexNode.getDefinition().getSecureFacetConfiguration())
);
PERF_LOGGER.end(f, -1, "facets retrieved");
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FacetHelper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FacetHelper.java?rev=1848182&r1=1848181&r2=1848182&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FacetHelper.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FacetHelper.java
Wed Dec 5 00:21:54 2018
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
+import
org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.SecureFacetConfiguration;
import org.apache.jackrabbit.oak.spi.query.QueryConstants;
import org.apache.jackrabbit.oak.spi.query.QueryIndex;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -35,7 +36,7 @@ import org.apache.lucene.facet.sortedset
import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetCounts;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.Sort;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -58,7 +59,8 @@ public class FacetHelper {
return new NodeStateFacetsConfig(definition);
}
- public static Facets getFacets(IndexSearcher searcher, Query query,
TopDocs docs, QueryIndex.IndexPlan plan, boolean secure) throws IOException {
+ public static Facets getFacets(IndexSearcher searcher, Query query,
QueryIndex.IndexPlan plan,
+ SecureFacetConfiguration
secureFacetConfiguration) throws IOException {
Facets facets = null;
@SuppressWarnings("unchecked")
List<String> facetFields = (List<String>)
plan.getAttribute(ATTR_FACET_FIELDS);
@@ -70,10 +72,23 @@ public class FacetHelper {
try {
DefaultSortedSetDocValuesReaderState state = new
DefaultSortedSetDocValuesReaderState(
searcher.getIndexReader(),
FieldNames.createFacetFieldName(facetField));
- FacetsCollector.search(searcher, query, 10,
facetsCollector);
- facetsMap.put(facetField, secure ?
- new FilteredSortedSetDocValuesFacetCounts(state,
facetsCollector, plan.getFilter(), docs) :
- new SortedSetDocValuesFacetCounts(state,
facetsCollector));
+ FacetsCollector.search(searcher, query, null,1,
Sort.INDEXORDER, facetsCollector);
+
+ switch (secureFacetConfiguration.getMode()) {
+ case INSECURE:
+ facets = new SortedSetDocValuesFacetCounts(state,
facetsCollector);
+ break;
+ case STATISTICAL:
+ facets = new
StatisticalSortedSetDocValuesFacetCounts(state, facetsCollector,
plan.getFilter(),
+ secureFacetConfiguration);
+ break;
+ case SECURE:
+ default:
+ facets = new
SecureSortedSetDocValuesFacetCounts(state, facetsCollector, plan.getFilter());
+ break;
+ }
+
+ facetsMap.put(facetField, facets);
} catch (IllegalArgumentException iae) {
LOGGER.warn("facets for {} not yet indexed", facetField);
Added:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SecureSortedSetDocValuesFacetCounts.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SecureSortedSetDocValuesFacetCounts.java?rev=1848182&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SecureSortedSetDocValuesFacetCounts.java
(added)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SecureSortedSetDocValuesFacetCounts.java
Wed Dec 5 00:21:54 2018
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene.util;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
+import org.apache.jackrabbit.oak.spi.query.Filter;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.facet.FacetResult;
+import org.apache.lucene.facet.FacetsCollector;
+import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.FacetsConfig;
+import org.apache.lucene.facet.LabelAndValue;
+import org.apache.lucene.facet.sortedset.DefaultSortedSetDocValuesReaderState;
+import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetCounts;
+import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.DocIdSet;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * ACL filtered version of {@link SortedSetDocValuesFacetCounts}
+ */
+class SecureSortedSetDocValuesFacetCounts extends
SortedSetDocValuesFacetCounts {
+
+ private final FacetsCollector facetsCollector;
+ private final Filter filter;
+ private final IndexReader reader;
+ private final SortedSetDocValuesReaderState state;
+ private FacetResult facetResult = null;
+
+ SecureSortedSetDocValuesFacetCounts(DefaultSortedSetDocValuesReaderState
state, FacetsCollector facetsCollector, Filter filter) throws IOException {
+ super(state, facetsCollector);
+ this.reader = state.origReader;
+ this.facetsCollector = facetsCollector;
+ this.filter = filter;
+ this.state = state;
+ }
+
+ @Override
+ public FacetResult getTopChildren(int topN, String dim, String... path)
throws IOException {
+ if (facetResult == null) {
+ facetResult = getTopChildren0(topN, dim, path);
+ }
+
+ return facetResult;
+ }
+
+ private FacetResult getTopChildren0(int topN, String dim, String... path)
throws IOException {
+ FacetResult topChildren = super.getTopChildren(topN, dim, path);
+
+ if (topChildren == null) {
+ return null;
+ }
+
+ InaccessibleFacetCountManager inaccessibleFacetCountManager =
+ new InaccessibleFacetCountManager(dim, reader, filter, state,
facetsCollector, topChildren.labelValues);
+ inaccessibleFacetCountManager.filterFacets();
+ LabelAndValue[] labelAndValues =
inaccessibleFacetCountManager.updateLabelAndValue();
+
+ int childCount = labelAndValues.length;
+ Number value = 0;
+ for (LabelAndValue lv : labelAndValues) {
+ value = value.longValue() + lv.value.longValue();
+ }
+
+ return new FacetResult(dim, path, value, labelAndValues, childCount);
+ }
+
+ static class InaccessibleFacetCountManager {
+ private final String dimension;
+ private final IndexReader reader;
+ private final Filter filter;
+ private final SortedSetDocValuesReaderState state;
+ private final FacetsCollector facetsCollector;
+ private final LabelAndValue[] labelAndValues;
+ private final Map<String, Integer> labelToIndexMap;
+ private final long[] inaccessibleCounts;
+
+ InaccessibleFacetCountManager(String dimension,
+ IndexReader reader, Filter filter,
SortedSetDocValuesReaderState state,
+ FacetsCollector facetsCollector,
LabelAndValue[] labelAndValues) {
+ this.dimension = dimension;
+ this.reader = reader;
+ this.filter = filter;
+ this.state = state;
+ this.facetsCollector = facetsCollector;
+ this.labelAndValues = labelAndValues;
+ inaccessibleCounts = new long[labelAndValues.length];
+
+ Map<String, Integer> map = Maps.newHashMap();
+ for (int i = 0; i < labelAndValues.length; i++) {
+ LabelAndValue lv = labelAndValues[i];
+ map.put(lv.label, i);
+ }
+ labelToIndexMap = Collections.unmodifiableMap(map);
+ }
+
+ void filterFacets() throws IOException {
+ List<MatchingDocs> matchingDocsList =
facetsCollector.getMatchingDocs();
+ for (MatchingDocs matchingDocs : matchingDocsList) {
+ DocIdSet bits = matchingDocs.bits;
+
+ DocIdSetIterator docIdSetIterator = bits.iterator();
+ int doc = docIdSetIterator.nextDoc();
+ while (doc != DocIdSetIterator.NO_MORE_DOCS) {
+ int docId = matchingDocs.context.docBase + doc;
+ filterFacet(docId);
+ doc = docIdSetIterator.nextDoc();
+ }
+ }
+ }
+
+ private void filterFacet(int docId) throws IOException {
+ Document document = reader.document(docId);
+
+ // filter using doc values (avoiding requiring stored values)
+ if
(!filter.isAccessible(document.getField(FieldNames.PATH).stringValue() + "/" +
dimension)) {
+
+ SortedSetDocValues docValues = state.getDocValues();
+ docValues.setDocument(docId);
+ TermsEnum termsEnum = docValues.termsEnum();
+
+ long ord = docValues.nextOrd();
+
+ while (ord != SortedSetDocValues.NO_MORE_ORDS) {
+ termsEnum.seekExact(ord);
+ String facetDVTerm = termsEnum.term().utf8ToString();
+ String [] facetDVDimPaths =
FacetsConfig.stringToPath(facetDVTerm);
+
+ // first element is dimention name
+ for (int i = 1; i < facetDVDimPaths.length; i++) {
+ markInaccessbile(facetDVDimPaths[i]);
+ }
+
+ ord = docValues.nextOrd();
+ }
+ }
+ }
+
+ void markInaccessbile(@NotNull String label) {
+ inaccessibleCounts[labelToIndexMap.get(label)]++;
+ }
+
+ LabelAndValue[] updateLabelAndValue() {
+ int numZeros = 0;
+
+ LabelAndValue[] newValues;
+
+ for (int i = 0; i < labelAndValues.length; i++) {
+ LabelAndValue lv = labelAndValues[i];
+ long inaccessibleCount =
inaccessibleCounts[labelToIndexMap.get(lv.label)];
+
+ if (inaccessibleCount > 0) {
+ long newValue = lv.value.longValue() - inaccessibleCount;
+
+ if (newValue <= 0) {
+ newValue = 0;
+ numZeros++;
+ }
+
+ labelAndValues[i] = new LabelAndValue(lv.label, newValue);
+ }
+ }
+
+ if (numZeros > 0) {
+ newValues = new LabelAndValue[labelAndValues.length -
numZeros];
+ int i = 0;
+ for (LabelAndValue lv : labelAndValues) {
+ if (lv.value.longValue() > 0) {
+ newValues[i++] = lv;
+ }
+ }
+ } else {
+ newValues = labelAndValues;
+ }
+
+ return newValues;
+ }
+ }
+}
\ No newline at end of file
Propchange:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SecureSortedSetDocValuesFacetCounts.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/StatisticalSortedSetDocValuesFacetCounts.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/StatisticalSortedSetDocValuesFacetCounts.java?rev=1848182&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/StatisticalSortedSetDocValuesFacetCounts.java
(added)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/StatisticalSortedSetDocValuesFacetCounts.java
Wed Dec 5 00:21:54 2018
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene.util;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.AbstractIterator;
+import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
+import
org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.SecureFacetConfiguration;
+import org.apache.jackrabbit.oak.spi.query.Filter;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.facet.FacetResult;
+import org.apache.lucene.facet.FacetsCollector;
+import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.LabelAndValue;
+import org.apache.lucene.facet.sortedset.DefaultSortedSetDocValuesReaderState;
+import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetCounts;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
+/**
+ * ACL filtered version of {@link SortedSetDocValuesFacetCounts}
+ */
+class StatisticalSortedSetDocValuesFacetCounts extends
SortedSetDocValuesFacetCounts {
+ private static final Logger LOG =
LoggerFactory.getLogger(StatisticalSortedSetDocValuesFacetCounts.class);
+
+ private final FacetsCollector facetsCollector;
+ private final Filter filter;
+ private final IndexReader reader;
+ private final SecureFacetConfiguration secureFacetConfiguration;
+ private FacetResult facetResult = null;
+
+
StatisticalSortedSetDocValuesFacetCounts(DefaultSortedSetDocValuesReaderState
state,
+ FacetsCollector
facetsCollector, Filter filter,
+ SecureFacetConfiguration
secureFacetConfiguration) throws IOException {
+ super(state, facetsCollector);
+ this.reader = state.origReader;
+ this.facetsCollector = facetsCollector;
+ this.filter = filter;
+ this.secureFacetConfiguration = secureFacetConfiguration;
+ }
+
+ @Override
+ public FacetResult getTopChildren(int topN, String dim, String... path)
throws IOException {
+ if (facetResult == null) {
+ facetResult = getTopChildren0(topN, dim, path);
+ }
+
+ return facetResult;
+ }
+
+ private FacetResult getTopChildren0(int topN, String dim, String... path)
throws IOException {
+ FacetResult topChildren = super.getTopChildren(topN, dim, path);
+
+ if (topChildren == null) {
+ return null;
+ }
+
+ LabelAndValue[] labelAndValues = topChildren.labelValues;
+
+ List<MatchingDocs> matchingDocsList =
facetsCollector.getMatchingDocs();
+
+ int hitCount = 0;
+ for (MatchingDocs matchingDocs : matchingDocsList) {
+ hitCount += matchingDocs.totalHits;
+ }
+ int sampleSize =
secureFacetConfiguration.getStatisticalFacetSampleSize();
+ long randomSeed = secureFacetConfiguration.getRandomSeed();
+
+ LOG.debug("Sampling facet dim {}; hitCount: {}, sampleSize: {}, seed:
{}", dim, hitCount, sampleSize, randomSeed);
+
+ Stopwatch w = Stopwatch.createStarted();
+ Iterator<Integer> docIterator =
getMatchingDocIterator(matchingDocsList);
+ Iterator<Integer> sampleIterator = docIterator;
+ if (sampleSize < hitCount) {
+ sampleIterator = getSampledMatchingDocIterator(docIterator,
randomSeed, hitCount, sampleSize);
+ } else {
+ sampleSize = hitCount;
+ }
+ int accessibleSampleCount = getAccessibleSampleCount(dim,
sampleIterator);
+ w.stop();
+
+ LOG.debug("Evaluated accessible samples {} in {}",
accessibleSampleCount, w);
+
+ labelAndValues = updateLabelAndValueIfRequired(labelAndValues,
sampleSize, accessibleSampleCount);
+
+ int childCount = labelAndValues.length;
+ Number value = 0;
+ for (LabelAndValue lv : labelAndValues) {
+ value = value.longValue() + lv.value.longValue();
+ }
+
+ return new FacetResult(dim, path, value, labelAndValues, childCount);
+ }
+
+ private Iterator<Integer> getMatchingDocIterator(final List<MatchingDocs>
matchingDocsList) {
+ Iterator<MatchingDocs> matchingDocsListIterator =
matchingDocsList.iterator();
+
+ return new AbstractIterator<Integer>() {
+ MatchingDocs matchingDocs = null;
+ DocIdSetIterator docIdSetIterator = null;
+ int nextDocId = NO_MORE_DOCS;
+ @Override
+ protected Integer computeNext() {
+ try {
+ loadNextMatchingDocsIfRequired();
+
+ if (nextDocId == NO_MORE_DOCS) {
+ return endOfData();
+ } else {
+ int ret = nextDocId;
+ nextDocId = docIdSetIterator.nextDoc();
+ return matchingDocs.context.docBase + ret;
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void loadNextMatchingDocsIfRequired() throws IOException {
+ while (nextDocId == NO_MORE_DOCS) {
+ if (matchingDocsListIterator.hasNext()) {
+ matchingDocs = matchingDocsListIterator.next();
+ docIdSetIterator = matchingDocs.bits.iterator();
+ nextDocId = docIdSetIterator.nextDoc();
+ } else {
+ return;
+ }
+ }
+ }
+ };
+ }
+
+ private Iterator<Integer> getSampledMatchingDocIterator(Iterator<Integer>
matchingDocs,
+ long randomdSeed,
int hitCount, int sampleSize) {
+ TapeSampling<Integer> tapeSampling = new TapeSampling<>(new
Random(randomdSeed), matchingDocs, hitCount, sampleSize);
+
+ return tapeSampling.getSamples();
+ }
+
+ private int getAccessibleSampleCount(String dim, Iterator<Integer>
sampleIterator) throws IOException {
+ int count = 0;
+ while (sampleIterator.hasNext()) {
+ int docId = sampleIterator.next();
+ Document doc = reader.document(docId);
+
+ if
(filter.isAccessible(doc.getField(FieldNames.PATH).stringValue() + "/" + dim)) {
+ count++;
+ }
+ }
+
+ return count;
+ }
+
+ private LabelAndValue[] updateLabelAndValueIfRequired(LabelAndValue[]
labelAndValues,
+ int sampleSize, int
accessibleCount) {
+ if (accessibleCount < sampleSize) {
+ int numZeros = 0;
+
+ LabelAndValue[] newValues;
+
+ {
+ LabelAndValue[] proportionedLVs = new
LabelAndValue[labelAndValues.length];
+
+ for (int i = 0; i < labelAndValues.length; i++) {
+ LabelAndValue lv = labelAndValues[i];
+ long count = Math.floorDiv(lv.value.longValue() *
accessibleCount, sampleSize);
+ if (count == 0) {
+ numZeros++;
+ }
+ proportionedLVs[i] = new LabelAndValue(lv.label, count);
+ }
+
+ labelAndValues = proportionedLVs;
+ }
+
+ if (numZeros > 0) {
+ newValues = new LabelAndValue[labelAndValues.length -
numZeros];
+ int i = 0;
+ for (LabelAndValue lv : labelAndValues) {
+ if (lv.value.longValue() > 0) {
+ newValues[i++] = lv;
+ }
+ }
+ } else {
+ newValues = labelAndValues;
+ }
+
+ return newValues;
+ } else {
+ return labelAndValues;
+ }
+ }
+}
\ No newline at end of file
Propchange:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/StatisticalSortedSetDocValuesFacetCounts.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java?rev=1848182&r1=1848181&r2=1848182&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
Wed Dec 5 00:21:54 2018
@@ -133,14 +133,7 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil.useV2;
import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorTest.createCal;
import static
org.apache.jackrabbit.oak.plugins.index.property.OrderedIndex.OrderDirection;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INCLUDE_PROPERTY_NAMES;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.ORDERED_PROP_NAMES;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_ANALYZED;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_NAME;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_NODE;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_NODE_SCOPE_INDEX;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_PROPERTY_INDEX;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_TYPE;
+import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.*;
import static
org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.INDEX_DEFINITION_NODE;
import static
org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty;
import static
org.apache.jackrabbit.oak.spi.filter.PathFilter.PROP_EXCLUDED_PATHS;
@@ -2908,6 +2901,7 @@ public class LucenePropertyIndexTest ext
idxb.indexRule("nt:base").property("foo").propertyIndex();
Tree idx = root.getTree("/").getChild("oak:index").addChild("test1");
idxb.build(idx);
+ idx.setProperty(PROP_RANDOM_SEED, new Random().nextLong());
root.commit();
AsyncIndexInfoService asyncService = new
AsyncIndexInfoServiceImpl(nodeStore);
@@ -3153,6 +3147,124 @@ public class LucenePropertyIndexTest ext
}
}
+ @Test
+ public void injectRandomSeedDuringReindex() throws Exception{
+ IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync();
+ idxb.indexRule("nt:base").property("foo").propertyIndex();
+ Tree idx = root.getTree("/").getChild("oak:index").addChild("test1");
+ idxb.build(idx);
+
+ root.commit();
+
+ // Push a change to get another indexing cycle run
+ root.getTree("/").addChild("force-index-run");
+ root.commit();
+
+ NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(),
idx.getPath());
+
+ long defSeed =
idxState.getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+ long clonedSeed =
idxState.getChildNode(INDEX_DEFINITION_NODE).getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+
+ assertTrue("Injected def (" + defSeed + ")and clone (" + clonedSeed +
" seeds aren't same", defSeed == clonedSeed);
+ }
+
+ @Test
+ public void injectRandomSeedDuringRefresh() throws Exception{
+ IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync();
+ idxb.indexRule("nt:base").property("foo").propertyIndex();
+ Tree idx = root.getTree("/").getChild("oak:index").addChild("test1");
+ idxb.build(idx);
+
+ root.commit();
+
+ long seed = new Random().nextLong();
+
+ // Push a change to get another indexing cycle run
+ idx.setProperty(PROP_RANDOM_SEED, seed);
+ idx.setProperty(PROP_REFRESH_DEFN, true);
+ root.commit();
+
+ NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(),
idx.getPath());
+
+ long defSeed =
idxState.getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+ long clonedSeed =
idxState.getChildNode(INDEX_DEFINITION_NODE).getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+
+ assertEquals("Random seed not updated", seed, defSeed);
+ assertTrue("Injected def (" + defSeed + ")and clone (" + clonedSeed +
" seeds aren't same", defSeed == clonedSeed);
+ }
+
+ @Test
+ public void injectRandomSeedDuringUpdate() throws Exception{
+ IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync();
+ idxb.indexRule("nt:base").property("foo").propertyIndex();
+ Tree idx = root.getTree("/").getChild("oak:index").addChild("test1");
+ idxb.build(idx);
+
+ root.commit();
+
+ long seed = new Random().nextLong();
+
+ // Update seed
+ idx.setProperty(PROP_RANDOM_SEED, seed);
+ root.commit();
+
+ NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(),
idx.getPath());
+
+ long defSeed =
idxState.getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+ long clonedSeed =
idxState.getChildNode(INDEX_DEFINITION_NODE).getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+
+ assertEquals("Random seed not updated", seed, defSeed);
+ assertTrue("Injected def (" + defSeed + ")and clone (" + clonedSeed +
" seeds aren't same", defSeed == clonedSeed);
+ }
+
+ @Test
+ public void injectGarbageSeed() throws Exception {
+ IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync();
+ idxb.indexRule("nt:base").property("foo").propertyIndex();
+ Tree idx = root.getTree("/").getChild("oak:index").addChild("test1");
+ idxb.build(idx);
+
+ root.commit();
+
+ long orignalSeed = NodeStateUtils.getNode(nodeStore.getRoot(),
idx.getPath())
+ .getProperty(PROP_RANDOM_SEED)
+ .getValue(Type.LONG);
+
+ // Update seed
+ idx.setProperty(PROP_RANDOM_SEED, "garbage");
+ root.commit();
+
+ NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(),
idx.getPath());
+
+ long defSeed =
idxState.getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+ long clonedSeed =
idxState.getChildNode(INDEX_DEFINITION_NODE).getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+
+ assertNotEquals("Random seed not updated", orignalSeed, defSeed);
+ assertNotEquals("Random seed updated to garbage", "garbage", defSeed);
+ assertTrue("Injected def (" + defSeed + ")and clone (" + clonedSeed +
" seeds aren't same", defSeed == clonedSeed);
+ }
+
+ @Test
+ public void injectStringLongSeed() throws Exception {
+ IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync();
+ idxb.indexRule("nt:base").property("foo").propertyIndex();
+ Tree idx = root.getTree("/").getChild("oak:index").addChild("test1");
+ idxb.build(idx);
+
+ root.commit();
+ // Update seed
+ idx.setProperty(PROP_RANDOM_SEED, "-12312");
+ root.commit();
+
+ NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(),
idx.getPath());
+
+ long defSeed =
idxState.getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+ long clonedSeed =
idxState.getChildNode(INDEX_DEFINITION_NODE).getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+
+ assertEquals("Random seed udpated to garbage", -12312, defSeed);
+ assertTrue("Injected def (" + defSeed + ")and clone (" + clonedSeed +
" seeds aren't same", defSeed == clonedSeed);
+ }
+
private void assertPlanAndQuery(String query, String planExpectation,
List<String> paths) {
assertPlanAndQuery(query, planExpectation, paths, false);
}
Added:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/SecureFacetTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/SecureFacetTest.java?rev=1848182&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/SecureFacetTest.java
(added)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/SecureFacetTest.java
Wed Dec 5 00:21:54 2018
@@ -0,0 +1,352 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import com.google.common.collect.Maps;
+import com.google.common.io.Closer;
+import
org.apache.jackrabbit.commons.jackrabbit.authorization.AccessControlUtils;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
+import org.apache.jackrabbit.oak.query.facet.FacetResult;
+import org.apache.jackrabbit.oak.query.facet.FacetResult.Facet;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.jetbrains.annotations.NotNull;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import javax.jcr.*;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+import javax.jcr.security.Privilege;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.apache.jackrabbit.commons.JcrUtils.getOrCreateByPath;
+import static org.apache.jackrabbit.oak.InitialContentHelper.INITIAL_CONTENT;
+import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
+import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.*;
+import static org.junit.Assert.*;
+
+public class SecureFacetTest {
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder(new
File("target"));
+
+ private Closer closer;
+
+ private Session session;
+ private Node indexNode;
+ private QueryManager qe;
+
+ private static final int NUM_LEAF_NODES =
STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT;
+ private static final int NUM_LABELS = 4;
+ private Map<String, Integer> actualLabelCount = Maps.newHashMap();
+ private Map<String, Integer> actualAclLabelCount = Maps.newHashMap();
+ private Map<String, Integer> actualAclPar1LabelCount = Maps.newHashMap();
+
+ @Before
+ public void setup() throws Exception {
+ closer = Closer.create();
+ createRepository();
+ createIndex();
+ }
+
+ private void createRepository() throws RepositoryException, IOException {
+ ExecutorService executorService = Executors.newFixedThreadPool(2);
+ closer.register(new ExecutorCloser(executorService));
+ IndexCopier copier = new IndexCopier(executorService,
temporaryFolder.getRoot());
+ LuceneIndexEditorProvider editorProvider = new
LuceneIndexEditorProvider(copier);
+ LuceneIndexProvider queryIndexProvider = new
LuceneIndexProvider(copier);
+ NodeStore nodeStore = new MemoryNodeStore(INITIAL_CONTENT);
+ Oak oak = new Oak(nodeStore)
+ .with((QueryIndexProvider) queryIndexProvider)
+ .with((Observer) queryIndexProvider)
+ .with(editorProvider);
+
+ Jcr jcr = new Jcr(oak);
+ @NotNull Repository repository = jcr.createRepository();
+
+ session = repository.login(new SimpleCredentials("admin",
"admin".toCharArray()), null);
+ closer.register(session::logout);
+
+ // we'd always query anonymously
+ Session anonSession = repository.login(new GuestCredentials());
+ closer.register(anonSession::logout);
+ qe = anonSession.getWorkspace().getQueryManager();
+ }
+
+ @After
+ public void after() throws IOException {
+ closer.close();
+ IndexDefinition.setDisableStoredIndexDefinition(false);
+ }
+
+ private void createIndex() throws RepositoryException {
+ IndexDefinitionBuilder idxBuilder = new IndexDefinitionBuilder();
+ idxBuilder.noAsync().evaluatePathRestrictions()
+ .indexRule("nt:base")
+ .property("cons")
+ .propertyIndex()
+ .property("foo")
+ .getBuilderTree().setProperty(PROP_FACETS, true);
+
+ indexNode = getOrCreateByPath("/oak:index", "nt:unstructured", session)
+ .addNode("index", INDEX_DEFINITIONS_NODE_TYPE);
+ idxBuilder.build(indexNode);
+ session.save();
+ }
+
+ private void createLargeDataset() throws RepositoryException {
+ Random rGen = new Random(42);
+ int[] labelCount = new int[NUM_LABELS];
+ int[] aclLabelCount = new int[NUM_LABELS];
+ int[] aclPar1LabelCount = new int[NUM_LABELS];
+
+ Node par = allow(getOrCreateByPath("/parent", "oak:Unstructured",
session));
+
+ for (int i = 0; i < 4; i++) {
+ Node subPar = par.addNode("par" + i);
+ for (int j = 0; j < NUM_LEAF_NODES; j++) {
+ Node child = subPar.addNode("c" + j);
+ child.setProperty("cons", "val");
+
+ // Add a random label out of "l0", "l1", "l2", "l3"
+ int labelNum = rGen.nextInt(NUM_LABELS);
+ child.setProperty("foo", "l" + labelNum);
+
+ labelCount[labelNum]++;
+ if (i != 0) {
+ aclLabelCount[labelNum]++;
+ }
+ if (i == 1) {
+ aclPar1LabelCount[labelNum]++;
+ }
+ }
+
+ // deny access for one sub-parent
+ if (i == 0) {
+ deny(subPar);
+ }
+ }
+
+ session.save();
+
+ for (int i = 0; i < labelCount.length; i++) {
+ actualLabelCount.put("l" + i, labelCount[i]);
+ actualAclLabelCount.put("l" + i, aclLabelCount[i]);
+ actualAclPar1LabelCount.put("l" + i, aclPar1LabelCount[i]);
+ }
+
+ assertNotEquals("Acl-ed and actual counts mustn't be same",
actualLabelCount, actualAclLabelCount);
+ }
+
+ @Test
+ public void secureFacets() throws Exception {
+ createLargeDataset();
+
+ assertEquals(actualAclLabelCount, getFacets());
+ }
+
+ @Test
+ public void secureFacets_withOneLabelInaccessible() throws Exception {
+ createLargeDataset();
+ Node inaccessibleChild =
deny(session.getNode("/parent").addNode("par4")).addNode("c0");
+ inaccessibleChild.setProperty("cons", "val");
+ inaccessibleChild.setProperty("foo", "l4");
+ session.save();
+
+ assertEquals(actualAclLabelCount, getFacets());
+ }
+
+ @Test
+ public void insecureFacets() throws Exception {
+ Node facetConfig = getOrCreateByPath(indexNode.getPath() + "/" +
FACETS, "nt:unstructured", session);
+ facetConfig.setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_INSECURE);
+ indexNode.setProperty(PROP_REFRESH_DEFN, true);
+ session.save();
+
+ createLargeDataset();
+
+ assertEquals(actualLabelCount, getFacets());
+ }
+
+ @Test
+ public void statisticalFacets() throws Exception {
+ Node facetConfig = getOrCreateByPath(indexNode.getPath() + "/" +
FACETS, "nt:unstructured", session);
+ facetConfig.setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_STATISTICAL);
+ indexNode.setProperty(PROP_REFRESH_DEFN, true);
+ session.save();
+
+ createLargeDataset();
+
+ Map<String, Integer> facets = getFacets();
+ assertEquals("Unexpected number of facets",
actualAclLabelCount.size(), facets.size());
+
+ for (Map.Entry<String, Integer> facet :
actualAclLabelCount.entrySet()) {
+ String facetLabel = facet.getKey();
+ int facetCount = facets.get(facetLabel);
+ float ratio = ((float)facetCount) / facet.getValue();
+ assertTrue("Facet count for label: " + facetLabel + " is outside
of 10% margin of error. " +
+ "Expected: " + facet.getValue() + "; Got: " +
facetCount + "; Ratio: " + ratio,
+ Math.abs(ratio - 1) < 0.1);
+ }
+ }
+
+ @Test
+ public void statisticalFacets_withHitCountSameAsSampleSize() throws
Exception {
+ Node facetConfig = getOrCreateByPath(indexNode.getPath() + "/" +
FACETS, "nt:unstructured", session);
+ facetConfig.setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_STATISTICAL);
+ indexNode.setProperty(PROP_REFRESH_DEFN, true);
+ session.save();
+
+ createLargeDataset();
+
+ Map<String, Integer> facets = getFacets("/parent/par1");
+ assertEquals("Unexpected number of facets",
actualAclPar1LabelCount.size(), facets.size());
+
+ for (Map.Entry<String, Integer> facet :
actualAclPar1LabelCount.entrySet()) {
+ String facetLabel = facet.getKey();
+ int facetCount = facets.get(facetLabel);
+ float ratio = ((float)facetCount) / facet.getValue();
+ assertTrue("Facet count for label: " + facetLabel + " is outside
of 10% margin of error. " +
+ "Expected: " + facet.getValue() + "; Got: " +
facetCount + "; Ratio: " + ratio,
+ Math.abs(ratio - 1) < 0.1);
+ }
+ }
+
+ @Test
+ public void statisticalFacets_withOneLabelInaccessible() throws Exception {
+ Node facetConfig = getOrCreateByPath(indexNode.getPath() + "/" +
FACETS, "nt:unstructured", session);
+ facetConfig.setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_STATISTICAL);
+ indexNode.setProperty(PROP_REFRESH_DEFN, true);
+ session.save();
+
+ createLargeDataset();
+ Node inaccessibleChild =
deny(session.getNode("/parent").addNode("par4")).addNode("c0");
+ inaccessibleChild.setProperty("cons", "val");
+ inaccessibleChild.setProperty("foo", "l4");
+ session.save();
+
+ Map<String, Integer> facets = getFacets();
+ assertEquals("Unexpected number of facets",
actualAclLabelCount.size(), facets.size());
+
+ for (Map.Entry<String, Integer> facet :
actualAclLabelCount.entrySet()) {
+ String facetLabel = facet.getKey();
+ int facetCount = facets.get(facetLabel);
+ float ratio = ((float)facetCount) / facet.getValue();
+ assertTrue("Facet count for label: " + facetLabel + " is outside
of 10% margin of error. " +
+ "Expected: " + facet.getValue() + "; Got: " +
facetCount + "; Ratio: " + ratio,
+ Math.abs(ratio - 1) < 0.1);
+ }
+ }
+
+ @Test
+ public void secureFacets_withAdminSession() throws Exception {
+ Node facetConfig = getOrCreateByPath(indexNode.getPath() + "/" +
FACETS, "nt:unstructured", session);
+ facetConfig.setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_INSECURE);
+ indexNode.setProperty(PROP_REFRESH_DEFN, true);
+ session.save();
+
+ createLargeDataset();
+
+ qe = session.getWorkspace().getQueryManager();
+
+ assertEquals(actualLabelCount, getFacets());
+ }
+
+ @Test
+ public void statisticalFacets_withAdminSession() throws Exception {
+ Node facetConfig = getOrCreateByPath(indexNode.getPath() + "/" +
FACETS, "nt:unstructured", session);
+ facetConfig.setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_STATISTICAL);
+ indexNode.setProperty(PROP_REFRESH_DEFN, true);
+ session.save();
+
+ createLargeDataset();
+
+ qe = session.getWorkspace().getQueryManager();
+
+ Map<String, Integer> facets = getFacets();
+ assertEquals("Unexpected number of facets", actualLabelCount.size(),
facets.size());
+
+ for (Map.Entry<String, Integer> facet : actualLabelCount.entrySet()) {
+ String facetLabel = facet.getKey();
+ int facetCount = facets.get(facetLabel);
+ float ratio = ((float)facetCount) / facet.getValue();
+ assertTrue("Facet count for label: " + facetLabel + " is outside
of 5% margin of error. " +
+ "Expected: " + facet.getValue() + "; Got: " +
facetCount + "; Ratio: " + ratio,
+ Math.abs(ratio - 1) < 0.05);
+ }
+ }
+
+ private Map<String, Integer> getFacets() throws Exception {
+ return getFacets(null);
+ }
+
+ private Map<String, Integer> getFacets(String path) throws Exception {
+ Map<String, Integer> map = Maps.newHashMap();
+
+ String pathCons = "";
+ if (path != null) {
+ pathCons = " AND ISDESCENDANTNODE('" + path + "')";
+ }
+ String query = "SELECT [rep:facet(foo)] FROM [nt:base] WHERE [cons] =
'val'" + pathCons;
+
+ Query q = qe.createQuery(query, Query.JCR_SQL2);
+ QueryResult queryResult = q.execute();
+
+ FacetResult facetResult = new FacetResult(queryResult);
+
+ Set<String> dims = facetResult.getDimensions();
+ for (String dim : dims) {
+ List<Facet> facets = facetResult.getFacets(dim);
+ for (Facet facet : facets) {
+ map.put(facet.getLabel(), facet.getCount());
+ }
+ }
+
+ return map;
+ }
+
+ @SuppressWarnings("UnusedReturnValue")
+ private Node deny(Node node) throws RepositoryException {
+ AccessControlUtils.deny(node, "anonymous", Privilege.JCR_ALL);
+ return node;
+ }
+
+ private Node allow(Node node) throws RepositoryException {
+ AccessControlUtils.allow(node, "anonymous", Privilege.JCR_READ);
+ return node;
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/SecureFacetTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-search/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/pom.xml?rev=1848182&r1=1848181&r2=1848182&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-search/pom.xml Wed Dec 5 00:21:54 2018
@@ -162,6 +162,13 @@
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-commons</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
Modified:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java?rev=1848182&r1=1848181&r2=1848182&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java
(original)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java
Wed Dec 5 00:21:54 2018
@@ -306,6 +306,15 @@ public interface FulltextIndexConstants
*/
String PROP_SECURE_FACETS = "secure";
+ String PROP_SECURE_FACETS_VALUE_INSECURE = "insecure";
+ String PROP_SECURE_FACETS_VALUE_STATISTICAL = "statistical";
+ String PROP_SECURE_FACETS_VALUE_SECURE = "secure";
+ String PROP_SECURE_FACETS_VALUE_JVM_PARAM = "oak.facets.secure";
+
+ String STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM =
"oak.facet.statistical.sampleSize";
+ String PROP_STATISTICAL_FACET_SAMPLE_SIZE = "sampleSize";
+ int STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT = 1000;
+
/**
* Optional (index definition) property indicating max number of facets
that will be retrieved
* in query
@@ -335,6 +344,13 @@ public interface FulltextIndexConstants
String PROP_REFRESH_DEFN = "refresh";
/**
+ * Long property that keep seed for random number generation. One example
usage of this is
+ * to randomly sample query results to statistically check for ACLs to
extrapolate facet
+ * counts
+ */
+ String PROP_RANDOM_SEED = "seed";
+
+ /**
* Boolean property to indicate that nodes nodetype matching indexRule name
* should be indexed
*/
Modified:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java?rev=1848182&r1=1848181&r2=1848182&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java
(original)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java
Wed Dec 5 00:21:54 2018
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
import java.util.regex.Pattern;
import javax.jcr.PropertyType;
@@ -79,25 +80,7 @@ import static org.apache.jackrabbit.oak.
import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEXING_MODE_NRT;
import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEXING_MODE_SYNC;
import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_COUNT;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.BLOB_SIZE;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.COMPAT_MODE;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.EVALUATE_PATH_RESTRICTION;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.EXCLUDE_PROPERTY_NAMES;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.EXPERIMENTAL_STORAGE;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.FACETS;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.FIELD_BOOST;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.FULL_TEXT_ENABLED;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INCLUDE_PROPERTY_NAMES;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INCLUDE_PROPERTY_TYPES;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INDEX_DATA_CHILD_NAME;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.ORDERED_PROP_NAMES;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_FACETS_TOP_CHILDREN;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_NODE;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_SECURE_FACETS;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.TIKA;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.TIKA_CONFIG;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.TIKA_MAPPED_TYPE;
-import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.TIKA_MIME_TYPES;
+import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.*;
import static
org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition.DEFAULT_BOOST;
import static
org.apache.jackrabbit.oak.plugins.index.search.util.ConfigUtil.getOptionalValue;
import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
@@ -244,7 +227,8 @@ public class IndexDefinition implements
private final boolean suggestAnalyzed;
- private final boolean secureFacets;
+ private final SecureFacetConfiguration secureFacets;
+ private final long randomSeed;
private final int numberOfTopFacets;
@@ -411,12 +395,21 @@ public class IndexDefinition implements
this.queryPaths = getOptionalStrings(defn, IndexConstants.QUERY_PATHS);
this.suggestAnalyzed = evaluateSuggestAnalyzed(defn, false);
+ {
+ PropertyState randomPS = defn.getProperty(PROP_RANDOM_SEED);
+ if (randomPS != null && randomPS.getType() == Type.LONG) {
+ randomSeed = randomPS.getValue(Type.LONG);
+ } else {
+ // create a random number
+ randomSeed = UUID.randomUUID().getMostSignificantBits();
+ }
+ }
if (defn.hasChildNode(FACETS)) {
NodeState facetsConfig = defn.getChildNode(FACETS);
- this.secureFacets = getOptionalValue(facetsConfig,
PROP_SECURE_FACETS, true);
+ this.secureFacets =
SecureFacetConfiguration.getInstance(randomSeed, facetsConfig);
this.numberOfTopFacets = getOptionalValue(facetsConfig,
PROP_FACETS_TOP_CHILDREN, DEFAULT_FACET_COUNT);
} else {
- this.secureFacets = true;
+ this.secureFacets =
SecureFacetConfiguration.getInstance(randomSeed, null);
this.numberOfTopFacets = DEFAULT_FACET_COUNT;
}
@@ -879,7 +872,7 @@ public class IndexDefinition implements
return suggestAnalyzed;
}
- public boolean isSecureFacets() {
+ public SecureFacetConfiguration getSecureFacetConfiguration() {
return secureFacets;
}
@@ -1814,4 +1807,97 @@ public class IndexDefinition implements
return new PropertyDefinition(rule, name, builder.getNodeState());
}
+ public static class SecureFacetConfiguration {
+ public enum MODE {
+ SECURE,
+ STATISTICAL,
+ INSECURE
+ }
+
+ private final long randomSeed;
+ private final MODE mode;
+ private final int statisticalFacetSampleSize;
+
+ SecureFacetConfiguration(long randomSeed, MODE mode, int
statisticalFacetSampleSize) {
+ this.randomSeed = randomSeed;
+ this.mode = mode;
+ this.statisticalFacetSampleSize = statisticalFacetSampleSize;
+ }
+
+ public MODE getMode() {
+ return mode;
+ }
+
+ public int getStatisticalFacetSampleSize() {
+ return statisticalFacetSampleSize;
+ }
+
+ public long getRandomSeed() {
+ return randomSeed;
+ }
+
+ static SecureFacetConfiguration getInstance(long randomSeed, NodeState
facetConfigRoot) {
+ if (facetConfigRoot == null) {
+ facetConfigRoot = EMPTY_NODE;
+ }
+
+ MODE mode = getMode(facetConfigRoot);
+ int statisticalFacetSampleSize;
+
+ if (mode == MODE.STATISTICAL) {
+ statisticalFacetSampleSize =
getStatisticalFacetSampleSize(facetConfigRoot);
+ } else {
+ statisticalFacetSampleSize = -1;
+ }
+
+ return new SecureFacetConfiguration(randomSeed, mode,
statisticalFacetSampleSize);
+ }
+
+ static MODE getMode(@NotNull final NodeState facetConfigRoot) {
+ PropertyState securePS =
facetConfigRoot.getProperty(PROP_SECURE_FACETS);
+ String modeString;
+
+ if (securePS != null) {
+ if(securePS.getType() == Type.BOOLEAN) {
+ // legacy secure config
+ boolean secure = securePS.getValue(Type.BOOLEAN);
+ return secure ? MODE.SECURE : MODE.INSECURE;
+ } else {
+ modeString = securePS.getValue(Type.STRING);
+ }
+ } else {
+ // use "default" just as placeholder. default case would do
the actual default eval
+ modeString =
System.getProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "default");
+ }
+
+ switch (modeString) {
+ case PROP_SECURE_FACETS_VALUE_INSECURE:
+ return MODE.INSECURE;
+ case PROP_SECURE_FACETS_VALUE_STATISTICAL:
+ return MODE.STATISTICAL;
+ case PROP_SECURE_FACETS_VALUE_SECURE:
+ default:
+ return MODE.SECURE;
+ }
+ }
+
+ static int getStatisticalFacetSampleSize(@NotNull final NodeState
facetConfigRoot) {
+ int statisticalFacetSampleSize = Integer.getInteger(
+ STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM,
+ STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT);
+ int statisticalFacetSampleSizePV =
getOptionalValue(facetConfigRoot,
+ PROP_STATISTICAL_FACET_SAMPLE_SIZE,
+ statisticalFacetSampleSize);
+
+ if (statisticalFacetSampleSizePV > 0) {
+ statisticalFacetSampleSize = statisticalFacetSampleSizePV;
+ }
+
+ if (statisticalFacetSampleSize < 0) {
+ statisticalFacetSampleSize =
STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT;
+ }
+
+ return statisticalFacetSampleSize;
+ }
+ }
}
Modified:
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java?rev=1848182&r1=1848181&r2=1848182&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java
(original)
+++
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java
Wed Dec 5 00:21:54 2018
@@ -20,8 +20,10 @@ package org.apache.jackrabbit.oak.plugin
import java.io.IOException;
import java.util.Calendar;
+import java.util.UUID;
import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.PerfLogger;
import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
@@ -44,6 +46,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.google.common.base.Preconditions.checkNotNull;
+import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_RANDOM_SEED;
import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_REFRESH_DEFN;
import static
org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.INDEX_DEFINITION_NODE;
@@ -248,6 +251,16 @@ public abstract class FulltextIndexEdito
private IndexDefinition createIndexDefinition(NodeState root, NodeBuilder
definition, IndexingContext
indexingContext, boolean asyncIndexing) {
+
+ // A good time to check and see if we want to inject our random
+ // seed - required due to OAK-7929 but could be useful otherwise too.
+ Long defRandom =
getLongPropertyOrNull(definition.getProperty(PROP_RANDOM_SEED));
+ if (defRandom == null) {
+ long seed = UUID.randomUUID().getMostSignificantBits();
+ definition.setProperty(PROP_RANDOM_SEED, seed);
+ defRandom = seed;
+ }
+
NodeState defnState = definition.getBaseState();
if (asyncIndexing && !IndexDefinition.isDisableStoredIndexDefinition()){
if (definition.getBoolean(PROP_REFRESH_DEFN)){
@@ -262,6 +275,17 @@ public abstract class FulltextIndexEdito
definition.setChildNode(INDEX_DEFINITION_NODE,
NodeStateCloner.cloneVisibleState(defnState));
log.info("Stored the cloned index definition for [{}]. Changes in
index definition would now only be " +
"effective post reindexing", indexingContext.getIndexPath());
+ } else {
+ // This is neither reindex nor refresh. So, let's update cloned def
with random seed
+ // if it doesn't match what's there in main definition
+ // Reindex would require another indexing cycle to update cloned def.
+ // Refresh would anyway is going to do a real clone so that's ok
+ NodeState clonedState = defnState.getChildNode(INDEX_DEFINITION_NODE);
+ Long clonedRandom =
getLongPropertyOrNull(clonedState.getProperty(PROP_RANDOM_SEED));
+
+ if (clonedRandom == null || clonedRandom != defRandom) {
+
definition.getChildNode(INDEX_DEFINITION_NODE).setProperty(PROP_RANDOM_SEED,
defRandom);
+ }
}
}
return newDefinitionBuilder()
@@ -270,4 +294,19 @@ public abstract class FulltextIndexEdito
.indexPath(indexingContext.getIndexPath())
.build();
}
+
+ private Long getLongPropertyOrNull(PropertyState propertyState) {
+ Long ret = null;
+
+ if (propertyState != null) {
+ try {
+ ret = propertyState.getValue(Type.LONG);
+ } catch (Exception e) {
+ // It's ok to ignore the exception, but let's log it to help debugging.
+ log.debug("Error occurred while reading property as long.", e);
+ }
+ }
+
+ return ret;
+ }
}
Added:
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinitionFacetConfigTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinitionFacetConfigTest.java?rev=1848182&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinitionFacetConfigTest.java
(added)
+++
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinitionFacetConfigTest.java
Wed Dec 5 00:21:54 2018
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.search;
+
+import org.apache.jackrabbit.oak.commons.junit.TemporarySystemProperty;
+import
org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.SecureFacetConfiguration;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.util.Random;
+
+import static
org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.*;
+import static
org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.SecureFacetConfiguration.MODE;
+import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+public class IndexDefinitionFacetConfigTest {
+
+ @Rule
+ public TemporarySystemProperty temporarySystemProperty = new
TemporarySystemProperty();
+
+ private NodeState root = EMPTY_NODE;
+
+ private final NodeBuilder builder = root.builder();
+
+ private static final long RANDOM_SEED = 1L;
+
+ @Test
+ public void defaultConfig() {
+ SecureFacetConfiguration config =
SecureFacetConfiguration.getInstance(RANDOM_SEED, root);
+
+ assertEquals(config.getMode(), MODE.SECURE);
+
+ int sampleSize = config.getStatisticalFacetSampleSize();
+ assertEquals(-1, sampleSize);
+ }
+
+ @Test
+ public void nonSecureConfigMode() {
+ SecureFacetConfiguration config;
+
+ builder.setProperty(PROP_SECURE_FACETS, "insecure");
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED,
builder.getNodeState());
+ assertEquals(-1, config.getStatisticalFacetSampleSize());
+
+ builder.setProperty(PROP_SECURE_FACETS, "statistical");
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED,
builder.getNodeState());
+ assertEquals(STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT,
config.getStatisticalFacetSampleSize());
+ }
+
+ @Test
+ public void systemPropSecureFacet() {
+ SecureFacetConfiguration config;
+
+ System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "random");
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED, root);
+ assertEquals(MODE.SECURE, config.getMode());
+
+ System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "secure");
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED, root);
+ assertEquals(MODE.SECURE, config.getMode());
+
+ System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "insecure");
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED, root);
+ assertEquals(MODE.INSECURE, config.getMode());
+
+ System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "insecure");
+ builder.setProperty(PROP_SECURE_FACETS, "secure");
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED,
builder.getNodeState());
+ assertEquals(MODE.SECURE, config.getMode());
+ }
+
+ @Test
+ public void systemPropSecureFacetStatisticalSampleSize() {
+ int sampleSize;
+
+ System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM,
PROP_SECURE_FACETS_VALUE_STATISTICAL);
+
+ System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "10");
+ sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED,
root).getStatisticalFacetSampleSize();
+ assertEquals(10, sampleSize);
+
+ System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "-10");
+ sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED,
root).getStatisticalFacetSampleSize();
+ assertEquals(STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT, sampleSize);
+
+ System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM,
"100000000000");
+ sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED,
root).getStatisticalFacetSampleSize();
+ assertEquals(STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT, sampleSize);
+ }
+
+ @Test
+ public void invalidSecureFacetSampleSize() {
+ int sampleSize;
+ NodeBuilder configBuilder =
builder.child("config").setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_STATISTICAL);
+ NodeState nodeState = configBuilder.getNodeState();
+
+ configBuilder.setProperty(PROP_STATISTICAL_FACET_SAMPLE_SIZE, -10);
+ sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED,
nodeState).getStatisticalFacetSampleSize();
+ assertEquals(STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT, sampleSize);
+
+ System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "10");
+ configBuilder.setProperty(PROP_STATISTICAL_FACET_SAMPLE_SIZE, -20);
+ nodeState = configBuilder.getNodeState();
+ sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED,
nodeState).getStatisticalFacetSampleSize();
+ assertEquals(10, sampleSize);
+
+ System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "-10");
+ configBuilder.setProperty(PROP_STATISTICAL_FACET_SAMPLE_SIZE, -20);
+ sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED,
nodeState).getStatisticalFacetSampleSize();
+ assertEquals(STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT, sampleSize);
+
+ System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "-10");
+ configBuilder.setProperty(PROP_STATISTICAL_FACET_SAMPLE_SIZE, 10);
+ nodeState = configBuilder.getNodeState();
+ sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED,
nodeState).getStatisticalFacetSampleSize();
+ assertEquals(10, sampleSize);
+ }
+
+ @Test
+ public void orderingOfOverrides() {
+ System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "insecure");
+ System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "10");
+
+ NodeState nodeState;
+ SecureFacetConfiguration config;
+ int sampleSize;
+
+ nodeState = builder.child("config1").setProperty(PROP_SECURE_FACETS,
"secure").getNodeState();
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED, nodeState);
+ assertEquals(MODE.SECURE, config.getMode());
+ assertEquals(-1, config.getStatisticalFacetSampleSize());
+
+ nodeState = builder.child("config2")
+ .setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_STATISTICAL)
+ .setProperty(PROP_STATISTICAL_FACET_SAMPLE_SIZE, 20)
+ .getNodeState();
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED, nodeState);
+ sampleSize = config.getStatisticalFacetSampleSize();
+ assertEquals(20, sampleSize);
+
+ nodeState = builder.child("config3")
+ .setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_STATISTICAL)
+ .getNodeState();
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED, nodeState);
+ sampleSize = config.getStatisticalFacetSampleSize();
+ assertEquals(10, sampleSize);
+ }
+
+ @Test
+ public void legacyConfig() {
+ NodeState ns = builder.setProperty(PROP_SECURE_FACETS,
true).getNodeState();
+ SecureFacetConfiguration config =
SecureFacetConfiguration.getInstance(RANDOM_SEED, ns);
+ assertEquals(MODE.SECURE, config.getMode());
+ assertEquals(-1, config.getStatisticalFacetSampleSize());
+
+ ns = builder.setProperty(PROP_SECURE_FACETS, false).getNodeState();
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED, ns);
+ assertEquals(MODE.INSECURE, config.getMode());
+ assertEquals(-1, config.getStatisticalFacetSampleSize());
+ }
+
+ @Test
+ public void absentFacetConfigNode() {
+ IndexDefinition idxDefn = new IndexDefinition(root, root, "/foo");
+
+ SecureFacetConfiguration config =
idxDefn.getSecureFacetConfiguration();
+
+ assertEquals(MODE.SECURE, config.getMode());
+ }
+
+ @Test
+ public void randomSeed() {
+ long seed = new Random().nextLong();
+ builder.setProperty(PROP_RANDOM_SEED, seed);
+ root = builder.getNodeState();
+ IndexDefinition idxDefn = new IndexDefinition(root, root, "/foo");
+
+ SecureFacetConfiguration config =
idxDefn.getSecureFacetConfiguration();
+
+ assertEquals(seed, config.getRandomSeed());
+ }
+
+ @Test
+ public void randomSeedWithoutOneInDef() {
+ long seed1 = new IndexDefinition(root, root,
"/foo").getSecureFacetConfiguration().getRandomSeed();
+ long seed2 = new IndexDefinition(root, root,
"/foo").getSecureFacetConfiguration().getRandomSeed();
+
+ assertNotEquals(seed1, seed2);
+ }
+}
\ No newline at end of file
Propchange:
jackrabbit/oak/trunk/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinitionFacetConfigTest.java
------------------------------------------------------------------------------
svn:eol-style = native
