Author: catholicon
Date: Wed Dec 5 14:49:15 2018
New Revision: 1848218
URL: http://svn.apache.org/viewvc?rev=1848218&view=rev
Log:
OAK-7929: Incorrect Facet Count With Large Dataset and ACLs (backport from
trunk)
Backport r1848181,r1848182,r1848191,r1848217 from trunk. The fixes cover
OAK-7930 as well.
Pasting commit message from those revisions below:
========== r1848181 ==========
OAK-7930: Add tape sampling
==============================
========== r1848182 ==========
OAK-7929: Incorrect Facet Count With Large Dataset and ACLs
* Allow secure and sampleSize under <index-def>/facets
* These can be overridden by JVM command line param
* Inject seed into index definition during indexing cycle
* Rename FilteredFacetCounts to SecureFacetCounts to serve for secure
configuration
* StatisticalSortedSetDocValueFacetCounts would do random sampling of
results wrt what ratio are accessible. Use this ratio to extrapolate
facet counts returned by index
* Add TemporarySystemProperty rule to ease writing tests which set system
property
==============================
========== r1848191 ==========
OAK-7929: Incorrect Facet Count With Large Dataset and ACLs
animal-sniffer failed on jdk11 while complaining that it couldn't find
Math.floorDiv(long, int). Thanks Michael Duerig for pointing out the
issue.
==============================
========== r1848217 ==========
OAK-7929: Incorrect Facet Count With Large Dataset and ACLs
While backporting to 1.6 branch there were test failure due to missing
code paths added during hybridV2 implementation (1.8). Changed random
seed injection slightly to make it compatible with 1.6 branch as well.
==============================
Added:
jackrabbit/oak/branches/1.8/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/junit/TemporarySystemProperty.java
(with props)
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SecureSortedSetDocValuesFacetCounts.java
(with props)
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/StatisticalSortedSetDocValuesFacetCounts.java
(with props)
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/TapeSampling.java
(with props)
jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionFacetConfigTest.java
(with props)
jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/SecureFacetTest.java
(with props)
jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/TapeSamplingTest.java
(with props)
Removed:
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FilteredSortedSetDocValuesFacetCounts.java
Modified:
jackrabbit/oak/branches/1.8/ (props changed)
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FacetHelper.java
jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
Propchange: jackrabbit/oak/branches/1.8/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 5 14:49:15 2018
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1820660-1820661,1820729,1820734,1820859,1820861,1820878,1820888,1820947,1821027,1821130,1821140-1821141,1821178,1821237,1821240,1821249,1821258,1821325,1821358,1821361-1821362,1821370,1821375,1821393,1821477,1821487,1821516,1821617,1821663,1821665,1821668,1821681,1821847,1821975-1821983,1822121,1822201,1822207,1822527,1822642,1822723,1822808,1822850,1822934,1823135,1823163,1823169,1823172,1823655,1823669,1824196,1824198,1824253,1824255,1824896,1824962,1825065,1825362,1825381,1825442,1825448,1825466,1825470-1825471,1825475,1825523,1825525,1825561,1825619-1825621,1825651,1825654,1825992,1826079,1826090,1826096,1826216,1826237,1826338,1826516,1826532,1826551,1826560,1826638,1826640,1826730,1826833,1826932,1826957,1827423,1827472,1827486,1827816,1827977,1828349,1828439,1828502,1828529,1828948,1829527,1829534,1829546,1829569,1829587,1829665,1829854,1829864,1829978,1829985,1829987,1829998,1830019,1830048,1830160,1830171,1830197,1830209,1830239,1830347,1830748,1830911
,1830923,1831157-1831158,1831163,1831190,1831374,1831560,1831689,1832258,1832376,1832379,1832535,1833308,1833347,1833833,1834112,1834117,1834287,1834291,1834302,1834326,1834328,1834336,1834428,1834468,1834483,1834610,1834648-1834649,1834681,1834823,1834857-1834858,1835060,1835518,1835521,1835635,1835642,1835780,1835819,1836082,1836121,1836167-1836168,1836170-1836187,1836189-1836196,1836206,1836487,1836493,1837057,1837274,1837296,1837326,1837475,1837503,1837547,1837569,1837600,1837657,1837718,1837998,1838076,1838637,1839549,1839570,1839637,1839746,1840019,1840024,1840031,1840226,1840455,1840462,1840574,1841314,1841352,1842089,1842677,1843175,1843222,1843231,1843398,1843618,1843652,1843911,1844325,1844549,1844625,1844627,1844642,1844728,1844775,1844932,1845135,1845336,1845405,1845415,1845730-1845731,1845863,1845865,1846057,1846617,1848073
+/jackrabbit/oak/trunk:1820660-1820661,1820729,1820734,1820859,1820861,1820878,1820888,1820947,1821027,1821130,1821140-1821141,1821178,1821237,1821240,1821249,1821258,1821325,1821358,1821361-1821362,1821370,1821375,1821393,1821477,1821487,1821516,1821617,1821663,1821665,1821668,1821681,1821847,1821975-1821983,1822121,1822201,1822207,1822527,1822642,1822723,1822808,1822850,1822934,1823135,1823163,1823169,1823172,1823655,1823669,1824196,1824198,1824253,1824255,1824896,1824962,1825065,1825362,1825381,1825442,1825448,1825466,1825470-1825471,1825475,1825523,1825525,1825561,1825619-1825621,1825651,1825654,1825992,1826079,1826090,1826096,1826216,1826237,1826338,1826516,1826532,1826551,1826560,1826638,1826640,1826730,1826833,1826932,1826957,1827423,1827472,1827486,1827816,1827977,1828349,1828439,1828502,1828529,1828948,1829527,1829534,1829546,1829569,1829587,1829665,1829854,1829864,1829978,1829985,1829987,1829998,1830019,1830048,1830160,1830171,1830197,1830209,1830239,1830347,1830748,1830911
,1830923,1831157-1831158,1831163,1831190,1831374,1831560,1831689,1832258,1832376,1832379,1832535,1833308,1833347,1833833,1834112,1834117,1834287,1834291,1834302,1834326,1834328,1834336,1834428,1834468,1834483,1834610,1834648-1834649,1834681,1834823,1834857-1834858,1835060,1835518,1835521,1835635,1835642,1835780,1835819,1836082,1836121,1836167-1836168,1836170-1836187,1836189-1836196,1836206,1836487,1836493,1837057,1837274,1837296,1837326,1837475,1837503,1837547,1837569,1837600,1837657,1837718,1837998,1838076,1838637,1839549,1839570,1839637,1839746,1840019,1840024,1840031,1840226,1840455,1840462,1840574,1841314,1841352,1842089,1842677,1843175,1843222,1843231,1843398,1843618,1843652,1843911,1844325,1844549,1844625,1844627,1844642,1844728,1844775,1844932,1845135,1845336,1845405,1845415,1845730-1845731,1845863,1845865,1846057,1846617,1848073,1848181-1848182,1848191,1848217
/jackrabbit/trunk:1345480
Added:
jackrabbit/oak/branches/1.8/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/junit/TemporarySystemProperty.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/junit/TemporarySystemProperty.java?rev=1848218&view=auto
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/junit/TemporarySystemProperty.java
(added)
+++
jackrabbit/oak/branches/1.8/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/junit/TemporarySystemProperty.java
Wed Dec 5 14:49:15 2018
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.commons.junit;
+
+import org.junit.rules.ExternalResource;
+
+import java.util.Properties;
+
+public class TemporarySystemProperty extends ExternalResource {
+
+ Properties oldProps = new Properties();
+
+ @Override
+ protected void before() {
+ oldProps.putAll(System.getProperties());
+ }
+
+ @Override
+ protected void after() {
+ System.setProperties(oldProps);
+ }
+}
Propchange:
jackrabbit/oak/branches/1.8/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/junit/TemporarySystemProperty.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java?rev=1848218&r1=1848217&r2=1848218&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
Wed Dec 5 14:49:15 2018
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import java.util.UUID;
import java.util.regex.Pattern;
import javax.jcr.PropertyType;
@@ -247,7 +248,8 @@ public final class IndexDefinition imple
private final boolean suggestAnalyzed;
- private final boolean secureFacets;
+ private final SecureFacetConfiguration secureFacets;
+ private final long randomSeed;
private final int numberOfTopFacets;
@@ -394,12 +396,21 @@ public final class IndexDefinition imple
this.saveDirListing = getOptionalValue(defn,
LuceneIndexConstants.SAVE_DIR_LISTING, true);
this.suggestAnalyzed = evaluateSuggestAnalyzed(defn, false);
+ {
+ PropertyState randomPS = defn.getProperty(PROP_RANDOM_SEED);
+ if (randomPS != null && randomPS.getType() == Type.LONG) {
+ randomSeed = randomPS.getValue(Type.LONG);
+ } else {
+ // create a random number
+ randomSeed = UUID.randomUUID().getMostSignificantBits();
+ }
+ }
if (defn.hasChildNode(FACETS)) {
NodeState facetsConfig = defn.getChildNode(FACETS);
- this.secureFacets = getOptionalValue(facetsConfig,
PROP_SECURE_FACETS, true);
+ this.secureFacets =
SecureFacetConfiguration.getInstance(randomSeed, facetsConfig);
this.numberOfTopFacets = getOptionalValue(facetsConfig,
PROP_FACETS_TOP_CHILDREN, DEFAULT_FACET_COUNT);
} else {
- this.secureFacets = true;
+ this.secureFacets =
SecureFacetConfiguration.getInstance(randomSeed, null);
this.numberOfTopFacets = DEFAULT_FACET_COUNT;
}
@@ -889,7 +900,7 @@ public final class IndexDefinition imple
return suggestAnalyzed;
}
- public boolean isSecureFacets() {
+ public SecureFacetConfiguration getSecureFacetConfiguration() {
return secureFacets;
}
@@ -1891,4 +1902,97 @@ public final class IndexDefinition imple
return new PropertyDefinition(rule, name, builder.getNodeState());
}
+ public static class SecureFacetConfiguration {
+ public enum MODE {
+ SECURE,
+ STATISTICAL,
+ INSECURE
+ }
+
+ private final long randomSeed;
+ private final MODE mode;
+ private final int statisticalFacetSampleSize;
+
+ SecureFacetConfiguration(long randomSeed, MODE mode, int
statisticalFacetSampleSize) {
+ this.randomSeed = randomSeed;
+ this.mode = mode;
+ this.statisticalFacetSampleSize = statisticalFacetSampleSize;
+ }
+
+ public MODE getMode() {
+ return mode;
+ }
+
+ public int getStatisticalFacetSampleSize() {
+ return statisticalFacetSampleSize;
+ }
+
+ public long getRandomSeed() {
+ return randomSeed;
+ }
+
+ static SecureFacetConfiguration getInstance(long randomSeed, NodeState
facetConfigRoot) {
+ if (facetConfigRoot == null) {
+ facetConfigRoot = EMPTY_NODE;
+ }
+
+ MODE mode = getMode(facetConfigRoot);
+ int statisticalFacetSampleSize;
+
+ if (mode == MODE.STATISTICAL) {
+ statisticalFacetSampleSize =
getStatisticalFacetSampleSize(facetConfigRoot);
+ } else {
+ statisticalFacetSampleSize = -1;
+ }
+
+ return new SecureFacetConfiguration(randomSeed, mode,
statisticalFacetSampleSize);
+ }
+
+ static MODE getMode(@NotNull final NodeState facetConfigRoot) {
+ PropertyState securePS =
facetConfigRoot.getProperty(PROP_SECURE_FACETS);
+ String modeString;
+
+ if (securePS != null) {
+ if(securePS.getType() == Type.BOOLEAN) {
+ // legacy secure config
+ boolean secure = securePS.getValue(Type.BOOLEAN);
+ return secure ? MODE.SECURE : MODE.INSECURE;
+ } else {
+ modeString = securePS.getValue(Type.STRING);
+ }
+ } else {
+ // use "default" just as placeholder. default case would do
the actual default eval
+ modeString =
System.getProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "default");
+ }
+
+ switch (modeString) {
+ case PROP_SECURE_FACETS_VALUE_INSECURE:
+ return MODE.INSECURE;
+ case PROP_SECURE_FACETS_VALUE_STATISTICAL:
+ return MODE.STATISTICAL;
+ case PROP_SECURE_FACETS_VALUE_SECURE:
+ default:
+ return MODE.SECURE;
+ }
+ }
+
+ static int getStatisticalFacetSampleSize(@NotNull final NodeState
facetConfigRoot) {
+ int statisticalFacetSampleSize = Integer.getInteger(
+ STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM,
+ STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT);
+ int statisticalFacetSampleSizePV =
getOptionalValue(facetConfigRoot,
+ PROP_STATISTICAL_FACET_SAMPLE_SIZE,
+ statisticalFacetSampleSize);
+
+ if (statisticalFacetSampleSizePV > 0) {
+ statisticalFacetSampleSize = statisticalFacetSampleSizePV;
+ }
+
+ if (statisticalFacetSampleSize < 0) {
+ statisticalFacetSampleSize =
STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT;
+ }
+
+ return statisticalFacetSampleSize;
+ }
+ }
}
Modified:
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java?rev=1848218&r1=1848217&r2=1848218&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
Wed Dec 5 14:49:15 2018
@@ -354,6 +354,15 @@ public interface LuceneIndexConstants {
*/
String FACETS = "facets";
+ String PROP_SECURE_FACETS_VALUE_INSECURE = "insecure";
+ String PROP_SECURE_FACETS_VALUE_STATISTICAL = "statistical";
+ String PROP_SECURE_FACETS_VALUE_SECURE = "secure";
+ String PROP_SECURE_FACETS_VALUE_JVM_PARAM = "oak.facets.secure";
+
+ String STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM =
"oak.facet.statistical.sampleSize";
+ String PROP_STATISTICAL_FACET_SAMPLE_SIZE = "sampleSize";
+ int STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT = 1000;
+
/**
* Optional property to set the suggest field to be analyzed and therefore
allow more fine
* grained and flexible suggestions.
@@ -394,6 +403,13 @@ public interface LuceneIndexConstants {
*/
String PROP_REFRESH_DEFN = "refresh";
+ /**
+ * Long property that keep seed for random number generation. One example
usage of this is
+ * to randomly sample query results to statistically check for ACLs to
extrapolate facet
+ * counts
+ */
+ String PROP_RANDOM_SEED = "seed";
+
/**
* Boolean property to indicate that nodes nodetype matching indexRule name
* should be indexed
Modified:
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java?rev=1848218&r1=1848217&r2=1848218&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
Wed Dec 5 14:49:15 2018
@@ -18,8 +18,10 @@ package org.apache.jackrabbit.oak.plugin
import java.io.IOException;
import java.util.Calendar;
+import java.util.UUID;
import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.PerfLogger;
import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
@@ -43,6 +45,7 @@ import org.slf4j.LoggerFactory;
import static com.google.common.base.Preconditions.checkNotNull;
import static
org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.INDEX_DEFINITION_NODE;
+import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PROP_RANDOM_SEED;
import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PROP_REFRESH_DEFN;
public class LuceneIndexEditorContext implements FacetsConfigProvider{
@@ -255,21 +258,58 @@ public class LuceneIndexEditorContext im
private static IndexDefinition createIndexDefinition(NodeState root,
NodeBuilder definition, IndexingContext
indexingContext, boolean asyncIndexing) {
NodeState defnState = definition.getBaseState();
- if (asyncIndexing &&
!IndexDefinition.isDisableStoredIndexDefinition()){
- if (definition.getBoolean(PROP_REFRESH_DEFN)){
- definition.removeProperty(PROP_REFRESH_DEFN);
- NodeState clonedState =
NodeStateCloner.cloneVisibleState(defnState);
- definition.setChildNode(INDEX_DEFINITION_NODE, clonedState);
- log.info("Refreshed the index definition for [{}]",
indexingContext.getIndexPath());
- if (log.isDebugEnabled()){
- log.debug("Updated index definition is {}",
NodeStateUtils.toString(clonedState));
+ if (asyncIndexing) {
+ // A good time to check and see if we want to inject our random
+ // seed - required due to OAK-7929 but could be useful otherwise
too.
+ Long defRandom =
getLongPropertyOrNull(definition.getProperty(PROP_RANDOM_SEED));
+ if (defRandom == null) {
+ long seed = UUID.randomUUID().getMostSignificantBits();
+ definition.setProperty(PROP_RANDOM_SEED, seed);
+ defRandom = seed;
+ }
+
+ if (!IndexDefinition.isDisableStoredIndexDefinition()) {
+ if (definition.getBoolean(PROP_REFRESH_DEFN)) {
+ definition.removeProperty(PROP_REFRESH_DEFN);
+ NodeState clonedState =
NodeStateCloner.cloneVisibleState(defnState);
+ definition.setChildNode(INDEX_DEFINITION_NODE,
clonedState);
+ log.info("Refreshed the index definition for [{}]",
indexingContext.getIndexPath());
+ if (log.isDebugEnabled()) {
+ log.debug("Updated index definition is {}",
NodeStateUtils.toString(clonedState));
+ }
+ } else if (!definition.hasChildNode(INDEX_DEFINITION_NODE)) {
+ definition.setChildNode(INDEX_DEFINITION_NODE,
NodeStateCloner.cloneVisibleState(defnState));
+ log.info("Stored the cloned index definition for [{}].
Changes in index definition would now only be " +
+ "effective post reindexing",
indexingContext.getIndexPath());
+ } else {
+ // This is neither reindex nor refresh. So, let's update
cloned def with random seed
+ // if it doesn't match what's there in main definition
+ // Reindex would require another indexing cycle to update
cloned def.
+ // Refresh would anyway is going to do a real clone so
that's ok
+ NodeState clonedState =
defnState.getChildNode(INDEX_DEFINITION_NODE);
+ Long clonedRandom =
getLongPropertyOrNull(clonedState.getProperty(PROP_RANDOM_SEED));
+
+ if (clonedRandom == null || clonedRandom != defRandom) {
+
definition.getChildNode(INDEX_DEFINITION_NODE).setProperty(PROP_RANDOM_SEED,
defRandom);
+ }
}
- } else if (!definition.hasChildNode(INDEX_DEFINITION_NODE)){
- definition.setChildNode(INDEX_DEFINITION_NODE,
NodeStateCloner.cloneVisibleState(defnState));
- log.info("Stored the cloned index definition for [{}]. Changes
in index definition would now only be " +
- "effective post reindexing",
indexingContext.getIndexPath());
}
}
return new IndexDefinition(root,
defnState,indexingContext.getIndexPath());
}
+
+ private static Long getLongPropertyOrNull(PropertyState propertyState) {
+ Long ret = null;
+
+ if (propertyState != null) {
+ try {
+ ret = propertyState.getValue(Type.LONG);
+ } catch (Exception e) {
+ // It's ok to ignore the exception, but let's log it to help
debugging.
+ log.debug("Error occurred while reading property as long.", e);
+ }
+ }
+
+ return ret;
+ }
}
Modified:
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1848218&r1=1848217&r2=1848218&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
Wed Dec 5 14:49:15 2018
@@ -442,7 +442,7 @@ public class LucenePropertyIndex impleme
long f = PERF_LOGGER.start();
if (!facetsInitialized) {
- facets = FacetHelper.getFacets(searcher,
query, docs, plan, indexNode.getDefinition().isSecureFacets());
+ facets = FacetHelper.getFacets(searcher,
query, plan, indexNode.getDefinition().getSecureFacetConfiguration());
facetsInitialized = true;
}
PERF_LOGGER.end(f, -1, "facets retrieved");
Modified:
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FacetHelper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FacetHelper.java?rev=1848218&r1=1848217&r2=1848218&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FacetHelper.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FacetHelper.java
Wed Dec 5 14:49:15 2018
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.SecureFacetConfiguration;
import org.apache.jackrabbit.oak.spi.query.QueryConstants;
import org.apache.jackrabbit.oak.spi.query.QueryIndex;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -35,7 +36,7 @@ import org.apache.lucene.facet.sortedset
import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetCounts;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.Sort;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -58,7 +59,8 @@ public class FacetHelper {
return new NodeStateFacetsConfig(definition);
}
- public static Facets getFacets(IndexSearcher searcher, Query query,
TopDocs docs, QueryIndex.IndexPlan plan, boolean secure) throws IOException {
+ public static Facets getFacets(IndexSearcher searcher, Query query,
QueryIndex.IndexPlan plan,
+ SecureFacetConfiguration
secureFacetConfiguration) throws IOException {
Facets facets = null;
@SuppressWarnings("unchecked")
List<String> facetFields = (List<String>)
plan.getAttribute(ATTR_FACET_FIELDS);
@@ -70,10 +72,23 @@ public class FacetHelper {
try {
DefaultSortedSetDocValuesReaderState state = new
DefaultSortedSetDocValuesReaderState(
searcher.getIndexReader(),
FieldNames.createFacetFieldName(facetField));
- FacetsCollector.search(searcher, query, 10,
facetsCollector);
- facetsMap.put(facetField, secure ?
- new FilteredSortedSetDocValuesFacetCounts(state,
facetsCollector, plan.getFilter(), docs) :
- new SortedSetDocValuesFacetCounts(state,
facetsCollector));
+ FacetsCollector.search(searcher, query, null,1,
Sort.INDEXORDER, facetsCollector);
+
+ switch (secureFacetConfiguration.getMode()) {
+ case INSECURE:
+ facets = new SortedSetDocValuesFacetCounts(state,
facetsCollector);
+ break;
+ case STATISTICAL:
+ facets = new
StatisticalSortedSetDocValuesFacetCounts(state, facetsCollector,
plan.getFilter(),
+ secureFacetConfiguration);
+ break;
+ case SECURE:
+ default:
+ facets = new
SecureSortedSetDocValuesFacetCounts(state, facetsCollector, plan.getFilter());
+ break;
+ }
+
+ facetsMap.put(facetField, facets);
} catch (IllegalArgumentException iae) {
LOGGER.warn("facets for {} not yet indexed", facetField);
Added:
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SecureSortedSetDocValuesFacetCounts.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SecureSortedSetDocValuesFacetCounts.java?rev=1848218&view=auto
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SecureSortedSetDocValuesFacetCounts.java
(added)
+++
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SecureSortedSetDocValuesFacetCounts.java
Wed Dec 5 14:49:15 2018
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene.util;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+import org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames;
+import org.apache.jackrabbit.oak.spi.query.Filter;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.facet.FacetResult;
+import org.apache.lucene.facet.FacetsCollector;
+import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.FacetsConfig;
+import org.apache.lucene.facet.LabelAndValue;
+import org.apache.lucene.facet.sortedset.DefaultSortedSetDocValuesReaderState;
+import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetCounts;
+import org.apache.lucene.facet.sortedset.SortedSetDocValuesReaderState;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.DocIdSet;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * ACL filtered version of {@link SortedSetDocValuesFacetCounts}
+ */
+class SecureSortedSetDocValuesFacetCounts extends
SortedSetDocValuesFacetCounts {
+
+ private final FacetsCollector facetsCollector;
+ private final Filter filter;
+ private final IndexReader reader;
+ private final SortedSetDocValuesReaderState state;
+ private FacetResult facetResult = null;
+
+ SecureSortedSetDocValuesFacetCounts(DefaultSortedSetDocValuesReaderState
state, FacetsCollector facetsCollector, Filter filter) throws IOException {
+ super(state, facetsCollector);
+ this.reader = state.origReader;
+ this.facetsCollector = facetsCollector;
+ this.filter = filter;
+ this.state = state;
+ }
+
+ @Override
+ public FacetResult getTopChildren(int topN, String dim, String... path)
throws IOException {
+ if (facetResult == null) {
+ facetResult = getTopChildren0(topN, dim, path);
+ }
+
+ return facetResult;
+ }
+
+ private FacetResult getTopChildren0(int topN, String dim, String... path)
throws IOException {
+ FacetResult topChildren = super.getTopChildren(topN, dim, path);
+
+ if (topChildren == null) {
+ return null;
+ }
+
+ InaccessibleFacetCountManager inaccessibleFacetCountManager =
+ new InaccessibleFacetCountManager(dim, reader, filter, state,
facetsCollector, topChildren.labelValues);
+ inaccessibleFacetCountManager.filterFacets();
+ LabelAndValue[] labelAndValues =
inaccessibleFacetCountManager.updateLabelAndValue();
+
+ int childCount = labelAndValues.length;
+ Number value = 0;
+ for (LabelAndValue lv : labelAndValues) {
+ value = value.longValue() + lv.value.longValue();
+ }
+
+ return new FacetResult(dim, path, value, labelAndValues, childCount);
+ }
+
+ static class InaccessibleFacetCountManager {
+ private final String dimension;
+ private final IndexReader reader;
+ private final Filter filter;
+ private final SortedSetDocValuesReaderState state;
+ private final FacetsCollector facetsCollector;
+ private final LabelAndValue[] labelAndValues;
+ private final Map<String, Integer> labelToIndexMap;
+ private final long[] inaccessibleCounts;
+
+ InaccessibleFacetCountManager(String dimension,
+ IndexReader reader, Filter filter,
SortedSetDocValuesReaderState state,
+ FacetsCollector facetsCollector,
LabelAndValue[] labelAndValues) {
+ this.dimension = dimension;
+ this.reader = reader;
+ this.filter = filter;
+ this.state = state;
+ this.facetsCollector = facetsCollector;
+ this.labelAndValues = labelAndValues;
+ inaccessibleCounts = new long[labelAndValues.length];
+
+ Map<String, Integer> map = Maps.newHashMap();
+ for (int i = 0; i < labelAndValues.length; i++) {
+ LabelAndValue lv = labelAndValues[i];
+ map.put(lv.label, i);
+ }
+ labelToIndexMap = Collections.unmodifiableMap(map);
+ }
+
+ void filterFacets() throws IOException {
+ List<MatchingDocs> matchingDocsList =
facetsCollector.getMatchingDocs();
+ for (MatchingDocs matchingDocs : matchingDocsList) {
+ DocIdSet bits = matchingDocs.bits;
+
+ DocIdSetIterator docIdSetIterator = bits.iterator();
+ int doc = docIdSetIterator.nextDoc();
+ while (doc != DocIdSetIterator.NO_MORE_DOCS) {
+ int docId = matchingDocs.context.docBase + doc;
+ filterFacet(docId);
+ doc = docIdSetIterator.nextDoc();
+ }
+ }
+ }
+
+ private void filterFacet(int docId) throws IOException {
+ Document document = reader.document(docId);
+
+ // filter using doc values (avoiding requiring stored values)
+ if
(!filter.isAccessible(document.getField(FieldNames.PATH).stringValue() + "/" +
dimension)) {
+
+ SortedSetDocValues docValues = state.getDocValues();
+ docValues.setDocument(docId);
+ TermsEnum termsEnum = docValues.termsEnum();
+
+ long ord = docValues.nextOrd();
+
+ while (ord != SortedSetDocValues.NO_MORE_ORDS) {
+ termsEnum.seekExact(ord);
+ String facetDVTerm = termsEnum.term().utf8ToString();
+ String [] facetDVDimPaths =
FacetsConfig.stringToPath(facetDVTerm);
+
+ // first element is dimention name
+ for (int i = 1; i < facetDVDimPaths.length; i++) {
+ markInaccessbile(facetDVDimPaths[i]);
+ }
+
+ ord = docValues.nextOrd();
+ }
+ }
+ }
+
+ void markInaccessbile(@NotNull String label) {
+ inaccessibleCounts[labelToIndexMap.get(label)]++;
+ }
+
+ LabelAndValue[] updateLabelAndValue() {
+ int numZeros = 0;
+
+ LabelAndValue[] newValues;
+
+ for (int i = 0; i < labelAndValues.length; i++) {
+ LabelAndValue lv = labelAndValues[i];
+ long inaccessibleCount =
inaccessibleCounts[labelToIndexMap.get(lv.label)];
+
+ if (inaccessibleCount > 0) {
+ long newValue = lv.value.longValue() - inaccessibleCount;
+
+ if (newValue <= 0) {
+ newValue = 0;
+ numZeros++;
+ }
+
+ labelAndValues[i] = new LabelAndValue(lv.label, newValue);
+ }
+ }
+
+ if (numZeros > 0) {
+ newValues = new LabelAndValue[labelAndValues.length -
numZeros];
+ int i = 0;
+ for (LabelAndValue lv : labelAndValues) {
+ if (lv.value.longValue() > 0) {
+ newValues[i++] = lv;
+ }
+ }
+ } else {
+ newValues = labelAndValues;
+ }
+
+ return newValues;
+ }
+ }
+}
\ No newline at end of file
Propchange:
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/SecureSortedSetDocValuesFacetCounts.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/StatisticalSortedSetDocValuesFacetCounts.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/StatisticalSortedSetDocValuesFacetCounts.java?rev=1848218&view=auto
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/StatisticalSortedSetDocValuesFacetCounts.java
(added)
+++
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/StatisticalSortedSetDocValuesFacetCounts.java
Wed Dec 5 14:49:15 2018
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene.util;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.AbstractIterator;
+import org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.SecureFacetConfiguration;
+import org.apache.jackrabbit.oak.spi.query.Filter;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.facet.FacetResult;
+import org.apache.lucene.facet.FacetsCollector;
+import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
+import org.apache.lucene.facet.LabelAndValue;
+import org.apache.lucene.facet.sortedset.DefaultSortedSetDocValuesReaderState;
+import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetCounts;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
+/**
+ * ACL filtered version of {@link SortedSetDocValuesFacetCounts}
+ */
+class StatisticalSortedSetDocValuesFacetCounts extends
SortedSetDocValuesFacetCounts {
+ private static final Logger LOG =
LoggerFactory.getLogger(StatisticalSortedSetDocValuesFacetCounts.class);
+
+ private final FacetsCollector facetsCollector;
+ private final Filter filter;
+ private final IndexReader reader;
+ private final SecureFacetConfiguration secureFacetConfiguration;
+ private FacetResult facetResult = null;
+
+
StatisticalSortedSetDocValuesFacetCounts(DefaultSortedSetDocValuesReaderState
state,
+ FacetsCollector
facetsCollector, Filter filter,
+ SecureFacetConfiguration
secureFacetConfiguration) throws IOException {
+ super(state, facetsCollector);
+ this.reader = state.origReader;
+ this.facetsCollector = facetsCollector;
+ this.filter = filter;
+ this.secureFacetConfiguration = secureFacetConfiguration;
+ }
+
+ @Override
+ public FacetResult getTopChildren(int topN, String dim, String... path)
throws IOException {
+ if (facetResult == null) {
+ facetResult = getTopChildren0(topN, dim, path);
+ }
+
+ return facetResult;
+ }
+
+ private FacetResult getTopChildren0(int topN, String dim, String... path)
throws IOException {
+ FacetResult topChildren = super.getTopChildren(topN, dim, path);
+
+ if (topChildren == null) {
+ return null;
+ }
+
+ LabelAndValue[] labelAndValues = topChildren.labelValues;
+
+ List<MatchingDocs> matchingDocsList =
facetsCollector.getMatchingDocs();
+
+ int hitCount = 0;
+ for (MatchingDocs matchingDocs : matchingDocsList) {
+ hitCount += matchingDocs.totalHits;
+ }
+ int sampleSize =
secureFacetConfiguration.getStatisticalFacetSampleSize();
+ long randomSeed = secureFacetConfiguration.getRandomSeed();
+
+ LOG.debug("Sampling facet dim {}; hitCount: {}, sampleSize: {}, seed:
{}", dim, hitCount, sampleSize, randomSeed);
+
+ Stopwatch w = Stopwatch.createStarted();
+ Iterator<Integer> docIterator =
getMatchingDocIterator(matchingDocsList);
+ Iterator<Integer> sampleIterator = docIterator;
+ if (sampleSize < hitCount) {
+ sampleIterator = getSampledMatchingDocIterator(docIterator,
randomSeed, hitCount, sampleSize);
+ } else {
+ sampleSize = hitCount;
+ }
+ int accessibleSampleCount = getAccessibleSampleCount(dim,
sampleIterator);
+ w.stop();
+
+ LOG.debug("Evaluated accessible samples {} in {}",
accessibleSampleCount, w);
+
+ labelAndValues = updateLabelAndValueIfRequired(labelAndValues,
sampleSize, accessibleSampleCount);
+
+ int childCount = labelAndValues.length;
+ Number value = 0;
+ for (LabelAndValue lv : labelAndValues) {
+ value = value.longValue() + lv.value.longValue();
+ }
+
+ return new FacetResult(dim, path, value, labelAndValues, childCount);
+ }
+
+ private Iterator<Integer> getMatchingDocIterator(final List<MatchingDocs>
matchingDocsList) {
+ Iterator<MatchingDocs> matchingDocsListIterator =
matchingDocsList.iterator();
+
+ return new AbstractIterator<Integer>() {
+ MatchingDocs matchingDocs = null;
+ DocIdSetIterator docIdSetIterator = null;
+ int nextDocId = NO_MORE_DOCS;
+ @Override
+ protected Integer computeNext() {
+ try {
+ loadNextMatchingDocsIfRequired();
+
+ if (nextDocId == NO_MORE_DOCS) {
+ return endOfData();
+ } else {
+ int ret = nextDocId;
+ nextDocId = docIdSetIterator.nextDoc();
+ return matchingDocs.context.docBase + ret;
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void loadNextMatchingDocsIfRequired() throws IOException {
+ while (nextDocId == NO_MORE_DOCS) {
+ if (matchingDocsListIterator.hasNext()) {
+ matchingDocs = matchingDocsListIterator.next();
+ docIdSetIterator = matchingDocs.bits.iterator();
+ nextDocId = docIdSetIterator.nextDoc();
+ } else {
+ return;
+ }
+ }
+ }
+ };
+ }
+
+ private Iterator<Integer> getSampledMatchingDocIterator(Iterator<Integer>
matchingDocs,
+ long randomdSeed,
int hitCount, int sampleSize) {
+ TapeSampling<Integer> tapeSampling = new TapeSampling<>(new
Random(randomdSeed), matchingDocs, hitCount, sampleSize);
+
+ return tapeSampling.getSamples();
+ }
+
+ private int getAccessibleSampleCount(String dim, Iterator<Integer>
sampleIterator) throws IOException {
+ int count = 0;
+ while (sampleIterator.hasNext()) {
+ int docId = sampleIterator.next();
+ Document doc = reader.document(docId);
+
+ if
(filter.isAccessible(doc.getField(FieldNames.PATH).stringValue() + "/" + dim)) {
+ count++;
+ }
+ }
+
+ return count;
+ }
+
+ private LabelAndValue[] updateLabelAndValueIfRequired(LabelAndValue[]
labelAndValues,
+ int sampleSize, int
accessibleCount) {
+ if (accessibleCount < sampleSize) {
+ int numZeros = 0;
+
+ LabelAndValue[] newValues;
+
+ {
+ LabelAndValue[] proportionedLVs = new
LabelAndValue[labelAndValues.length];
+
+ for (int i = 0; i < labelAndValues.length; i++) {
+ LabelAndValue lv = labelAndValues[i];
+ long count = lv.value.longValue() * accessibleCount /
sampleSize;
+ if (count == 0) {
+ numZeros++;
+ }
+ proportionedLVs[i] = new LabelAndValue(lv.label, count);
+ }
+
+ labelAndValues = proportionedLVs;
+ }
+
+ if (numZeros > 0) {
+ newValues = new LabelAndValue[labelAndValues.length -
numZeros];
+ int i = 0;
+ for (LabelAndValue lv : labelAndValues) {
+ if (lv.value.longValue() > 0) {
+ newValues[i++] = lv;
+ }
+ }
+ } else {
+ newValues = labelAndValues;
+ }
+
+ return newValues;
+ } else {
+ return labelAndValues;
+ }
+ }
+}
\ No newline at end of file
Propchange:
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/StatisticalSortedSetDocValuesFacetCounts.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/TapeSampling.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/TapeSampling.java?rev=1848218&view=auto
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/TapeSampling.java
(added)
+++
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/TapeSampling.java
Wed Dec 5 14:49:15 2018
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene.util;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.AbstractIterator;
+
+import java.util.Iterator;
+import java.util.Random;
+
+/**
+ * Sampling algorithm that picks 'k' random samples from streaming input.
+ * The algorithm would maintain 'k/N' probability to pick any of the item
+ * where 'N' is the number of items seen currently.
+ *
+ * While the input could be streaming, the algorithm requires {@code N} to be
known
+ * before hand.
+ *
+ * The algorithm produces random saamples without replacement and hence has
O(1) extra
+ * memory complexity
+ *
+ * Implementation inspired from "JONES,T.G. A note on sampling a tape file"
+ * (https://dl.acm.org/citation.cfm?id=368159)
+ */
+public class TapeSampling<T> {
+ private final Random rGen;
+ private final Iterator<T> input;
+ private final int N;
+ private final int k;
+
+ public TapeSampling(final Random rGen, final Iterator<T> input, final int
N, final int k) {
+ this.rGen = rGen;
+ this.input = input;
+ this.N = N;
+ this.k = k;
+ }
+
+ public Iterator<T> getSamples() {
+ return new AbstractIterator<T>() {
+ int sampled = 0;
+ int seen = 0;
+
+ @Override
+ protected T computeNext() {
+ if (sampled == k) {
+ return endOfData();
+ }
+
+ while (true) {
+ Preconditions.checkArgument(input.hasNext(),
+ "Not enough input items provided. Declared: " + N
+ "; got " + seen + "; sampled: " + sampled);
+
+ T i = input.next();
+
+ int r = rGen.nextInt(N - seen) + 1;
+ seen++;
+
+ if (r <= k - sampled) {
+ sampled++;
+ return i;
+ }
+ }
+ }
+ };
+ }
+}
Propchange:
jackrabbit/oak/branches/1.8/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/TapeSampling.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionFacetConfigTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionFacetConfigTest.java?rev=1848218&view=auto
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionFacetConfigTest.java
(added)
+++
jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionFacetConfigTest.java
Wed Dec 5 14:49:15 2018
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import org.apache.jackrabbit.oak.commons.junit.TemporarySystemProperty;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.SecureFacetConfiguration;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.util.Random;
+
+import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.*;
+import static
org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.SecureFacetConfiguration.MODE;
+import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+public class IndexDefinitionFacetConfigTest {
+
+ @Rule
+ public TemporarySystemProperty temporarySystemProperty = new
TemporarySystemProperty();
+
+ private NodeState root = EMPTY_NODE;
+
+ private final NodeBuilder builder = root.builder();
+
+ private static final long RANDOM_SEED = 1L;
+
+ @Test
+ public void defaultConfig() {
+ SecureFacetConfiguration config =
SecureFacetConfiguration.getInstance(RANDOM_SEED, root);
+
+ assertEquals(config.getMode(), MODE.SECURE);
+
+ int sampleSize = config.getStatisticalFacetSampleSize();
+ assertEquals(-1, sampleSize);
+ }
+
+ @Test
+ public void nonSecureConfigMode() {
+ SecureFacetConfiguration config;
+
+ builder.setProperty(PROP_SECURE_FACETS, "insecure");
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED,
builder.getNodeState());
+ assertEquals(-1, config.getStatisticalFacetSampleSize());
+
+ builder.setProperty(PROP_SECURE_FACETS, "statistical");
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED,
builder.getNodeState());
+ assertEquals(STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT,
config.getStatisticalFacetSampleSize());
+ }
+
+ @Test
+ public void systemPropSecureFacet() {
+ SecureFacetConfiguration config;
+
+ System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "random");
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED, root);
+ assertEquals(MODE.SECURE, config.getMode());
+
+ System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "secure");
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED, root);
+ assertEquals(MODE.SECURE, config.getMode());
+
+ System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "insecure");
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED, root);
+ assertEquals(MODE.INSECURE, config.getMode());
+
+ System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "insecure");
+ builder.setProperty(PROP_SECURE_FACETS, "secure");
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED,
builder.getNodeState());
+ assertEquals(MODE.SECURE, config.getMode());
+ }
+
+ @Test
+ public void systemPropSecureFacetStatisticalSampleSize() {
+ int sampleSize;
+
+ System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM,
PROP_SECURE_FACETS_VALUE_STATISTICAL);
+
+ System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "10");
+ sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED,
root).getStatisticalFacetSampleSize();
+ assertEquals(10, sampleSize);
+
+ System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "-10");
+ sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED,
root).getStatisticalFacetSampleSize();
+ assertEquals(STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT, sampleSize);
+
+ System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM,
"100000000000");
+ sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED,
root).getStatisticalFacetSampleSize();
+ assertEquals(STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT, sampleSize);
+ }
+
+ @Test
+ public void invalidSecureFacetSampleSize() {
+ int sampleSize;
+ NodeBuilder configBuilder =
builder.child("config").setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_STATISTICAL);
+ NodeState nodeState = configBuilder.getNodeState();
+
+ configBuilder.setProperty(PROP_STATISTICAL_FACET_SAMPLE_SIZE, -10);
+ sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED,
nodeState).getStatisticalFacetSampleSize();
+ assertEquals(STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT, sampleSize);
+
+ System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "10");
+ configBuilder.setProperty(PROP_STATISTICAL_FACET_SAMPLE_SIZE, -20);
+ nodeState = configBuilder.getNodeState();
+ sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED,
nodeState).getStatisticalFacetSampleSize();
+ assertEquals(10, sampleSize);
+
+ System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "-10");
+ configBuilder.setProperty(PROP_STATISTICAL_FACET_SAMPLE_SIZE, -20);
+ sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED,
nodeState).getStatisticalFacetSampleSize();
+ assertEquals(STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT, sampleSize);
+
+ System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "-10");
+ configBuilder.setProperty(PROP_STATISTICAL_FACET_SAMPLE_SIZE, 10);
+ nodeState = configBuilder.getNodeState();
+ sampleSize = SecureFacetConfiguration.getInstance(RANDOM_SEED,
nodeState).getStatisticalFacetSampleSize();
+ assertEquals(10, sampleSize);
+ }
+
+ @Test
+ public void orderingOfOverrides() {
+ System.setProperty(PROP_SECURE_FACETS_VALUE_JVM_PARAM, "insecure");
+ System.setProperty(STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM, "10");
+
+ NodeState nodeState;
+ SecureFacetConfiguration config;
+ int sampleSize;
+
+ nodeState = builder.child("config1").setProperty(PROP_SECURE_FACETS,
"secure").getNodeState();
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED, nodeState);
+ assertEquals(MODE.SECURE, config.getMode());
+ assertEquals(-1, config.getStatisticalFacetSampleSize());
+
+ nodeState = builder.child("config2")
+ .setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_STATISTICAL)
+ .setProperty(PROP_STATISTICAL_FACET_SAMPLE_SIZE, 20)
+ .getNodeState();
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED, nodeState);
+ sampleSize = config.getStatisticalFacetSampleSize();
+ assertEquals(20, sampleSize);
+
+ nodeState = builder.child("config3")
+ .setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_STATISTICAL)
+ .getNodeState();
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED, nodeState);
+ sampleSize = config.getStatisticalFacetSampleSize();
+ assertEquals(10, sampleSize);
+ }
+
+ @Test
+ public void legacyConfig() {
+ NodeState ns = builder.setProperty(PROP_SECURE_FACETS,
true).getNodeState();
+ SecureFacetConfiguration config =
SecureFacetConfiguration.getInstance(RANDOM_SEED, ns);
+ assertEquals(MODE.SECURE, config.getMode());
+ assertEquals(-1, config.getStatisticalFacetSampleSize());
+
+ ns = builder.setProperty(PROP_SECURE_FACETS, false).getNodeState();
+ config = SecureFacetConfiguration.getInstance(RANDOM_SEED, ns);
+ assertEquals(MODE.INSECURE, config.getMode());
+ assertEquals(-1, config.getStatisticalFacetSampleSize());
+ }
+
+ @Test
+ public void absentFacetConfigNode() {
+ IndexDefinition idxDefn = new IndexDefinition(root, root, "/foo");
+
+ SecureFacetConfiguration config =
idxDefn.getSecureFacetConfiguration();
+
+ assertEquals(MODE.SECURE, config.getMode());
+ }
+
+ @Test
+ public void randomSeed() {
+ long seed = new Random().nextLong();
+ builder.setProperty(PROP_RANDOM_SEED, seed);
+ root = builder.getNodeState();
+ IndexDefinition idxDefn = new IndexDefinition(root, root, "/foo");
+
+ SecureFacetConfiguration config =
idxDefn.getSecureFacetConfiguration();
+
+ assertEquals(seed, config.getRandomSeed());
+ }
+
+ @Test
+ public void randomSeedWithoutOneInDef() {
+ long seed1 = new IndexDefinition(root, root,
"/foo").getSecureFacetConfiguration().getRandomSeed();
+ long seed2 = new IndexDefinition(root, root,
"/foo").getSecureFacetConfiguration().getRandomSeed();
+
+ assertNotEquals(seed1, seed2);
+ }
+}
\ No newline at end of file
Propchange:
jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionFacetConfigTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java?rev=1848218&r1=1848217&r2=1848218&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
Wed Dec 5 14:49:15 2018
@@ -48,6 +48,8 @@ import static org.apache.jackrabbit.oak.
import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PROP_NODE;
import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PROP_NODE_SCOPE_INDEX;
import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PROP_PROPERTY_INDEX;
+import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PROP_REFRESH_DEFN;
+import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PROP_RANDOM_SEED;
import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PROP_TYPE;
import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TIKA;
import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorTest.createCal;
@@ -2709,6 +2711,7 @@ public class LucenePropertyIndexTest ext
idxb.indexRule("nt:base").property("foo").propertyIndex();
Tree idx = root.getTree("/").getChild("oak:index").addChild("test1");
idxb.build(idx);
+ idx.setProperty(PROP_RANDOM_SEED, new Random().nextLong());
root.commit();
AsyncIndexInfoService asyncService = new
AsyncIndexInfoServiceImpl(nodeStore);
@@ -2854,6 +2857,124 @@ public class LucenePropertyIndexTest ext
}
}
+ @Test
+ public void injectRandomSeedDuringReindex() throws Exception{
+ IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync();
+ idxb.indexRule("nt:base").property("foo").propertyIndex();
+ Tree idx = root.getTree("/").getChild("oak:index").addChild("test1");
+ idxb.build(idx);
+
+ root.commit();
+
+ // Push a change to get another indexing cycle run
+ root.getTree("/").addChild("force-index-run");
+ root.commit();
+
+ NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(),
idx.getPath());
+
+ long defSeed =
idxState.getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+ long clonedSeed =
idxState.getChildNode(INDEX_DEFINITION_NODE).getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+
+ assertTrue("Injected def (" + defSeed + ")and clone (" + clonedSeed +
" seeds aren't same", defSeed == clonedSeed);
+ }
+
+ @Test
+ public void injectRandomSeedDuringRefresh() throws Exception{
+ IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync();
+ idxb.indexRule("nt:base").property("foo").propertyIndex();
+ Tree idx = root.getTree("/").getChild("oak:index").addChild("test1");
+ idxb.build(idx);
+
+ root.commit();
+
+ long seed = new Random().nextLong();
+
+ // Push a change to get another indexing cycle run
+ idx.setProperty(PROP_RANDOM_SEED, seed);
+ idx.setProperty(PROP_REFRESH_DEFN, true);
+ root.commit();
+
+ NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(),
idx.getPath());
+
+ long defSeed =
idxState.getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+ long clonedSeed =
idxState.getChildNode(INDEX_DEFINITION_NODE).getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+
+ assertEquals("Random seed not updated", seed, defSeed);
+ assertTrue("Injected def (" + defSeed + ")and clone (" + clonedSeed +
" seeds aren't same", defSeed == clonedSeed);
+ }
+
+ @Test
+ public void injectRandomSeedDuringUpdate() throws Exception{
+ IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync();
+ idxb.indexRule("nt:base").property("foo").propertyIndex();
+ Tree idx = root.getTree("/").getChild("oak:index").addChild("test1");
+ idxb.build(idx);
+
+ root.commit();
+
+ long seed = new Random().nextLong();
+
+ // Update seed
+ idx.setProperty(PROP_RANDOM_SEED, seed);
+ root.commit();
+
+ NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(),
idx.getPath());
+
+ long defSeed =
idxState.getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+ long clonedSeed =
idxState.getChildNode(INDEX_DEFINITION_NODE).getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+
+ assertEquals("Random seed not updated", seed, defSeed);
+ assertTrue("Injected def (" + defSeed + ")and clone (" + clonedSeed +
" seeds aren't same", defSeed == clonedSeed);
+ }
+
+ @Test
+ public void injectGarbageSeed() throws Exception {
+ IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync();
+ idxb.indexRule("nt:base").property("foo").propertyIndex();
+ Tree idx = root.getTree("/").getChild("oak:index").addChild("test1");
+ idxb.build(idx);
+
+ root.commit();
+
+ long orignalSeed = NodeStateUtils.getNode(nodeStore.getRoot(),
idx.getPath())
+ .getProperty(PROP_RANDOM_SEED)
+ .getValue(Type.LONG);
+
+ // Update seed
+ idx.setProperty(PROP_RANDOM_SEED, "garbage");
+ root.commit();
+
+ NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(),
idx.getPath());
+
+ long defSeed =
idxState.getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+ long clonedSeed =
idxState.getChildNode(INDEX_DEFINITION_NODE).getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+
+ assertNotEquals("Random seed not updated", orignalSeed, defSeed);
+ assertNotEquals("Random seed updated to garbage", "garbage", defSeed);
+ assertTrue("Injected def (" + defSeed + ")and clone (" + clonedSeed +
" seeds aren't same", defSeed == clonedSeed);
+ }
+
+ @Test
+ public void injectStringLongSeed() throws Exception {
+ IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync();
+ idxb.indexRule("nt:base").property("foo").propertyIndex();
+ Tree idx = root.getTree("/").getChild("oak:index").addChild("test1");
+ idxb.build(idx);
+
+ root.commit();
+ // Update seed
+ idx.setProperty(PROP_RANDOM_SEED, "-12312");
+ root.commit();
+
+ NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(),
idx.getPath());
+
+ long defSeed =
idxState.getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+ long clonedSeed =
idxState.getChildNode(INDEX_DEFINITION_NODE).getProperty(PROP_RANDOM_SEED).getValue(Type.LONG);
+
+ assertEquals("Random seed udpated to garbage", -12312, defSeed);
+ assertTrue("Injected def (" + defSeed + ")and clone (" + clonedSeed +
" seeds aren't same", defSeed == clonedSeed);
+ }
+
private void assertPlanAndQuery(String query, String planExpectation,
List<String> paths) {
assertPlanAndQuery(query, planExpectation, paths, false);
}
Added:
jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/SecureFacetTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/SecureFacetTest.java?rev=1848218&view=auto
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/SecureFacetTest.java
(added)
+++
jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/SecureFacetTest.java
Wed Dec 5 14:49:15 2018
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import com.google.common.collect.Maps;
+import com.google.common.io.Closer;
+import
org.apache.jackrabbit.commons.jackrabbit.authorization.AccessControlUtils;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
+import org.apache.jackrabbit.oak.query.facet.FacetResult;
+import org.apache.jackrabbit.oak.query.facet.FacetResult.Facet;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.jetbrains.annotations.NotNull;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import javax.jcr.*;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+import javax.jcr.security.Privilege;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.apache.jackrabbit.commons.JcrUtils.getOrCreateByPath;
+import static org.apache.jackrabbit.oak.InitialContent.INITIAL_CONTENT;
+import static
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
+import static
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.*;
+import static org.junit.Assert.*;
+
+public class SecureFacetTest {
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder(new
File("target"));
+
+ private Closer closer;
+
+ private Session session;
+ private Node indexNode;
+ private QueryManager qe;
+
+ private static final int NUM_LEAF_NODES =
STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT;
+ private static final int NUM_LABELS = 4;
+ private Map<String, Integer> actualLabelCount = Maps.newHashMap();
+ private Map<String, Integer> actualAclLabelCount = Maps.newHashMap();
+ private Map<String, Integer> actualAclPar1LabelCount = Maps.newHashMap();
+
+ @Before
+ public void setup() throws Exception {
+ closer = Closer.create();
+ createRepository();
+ createIndex();
+ }
+
+ private void createRepository() throws RepositoryException, IOException {
+ ExecutorService executorService = Executors.newFixedThreadPool(2);
+ closer.register(new ExecutorCloser(executorService));
+ IndexCopier copier = new IndexCopier(executorService,
temporaryFolder.getRoot());
+ LuceneIndexEditorProvider editorProvider = new
LuceneIndexEditorProvider(copier);
+ LuceneIndexProvider queryIndexProvider = new
LuceneIndexProvider(copier);
+ NodeStore nodeStore = new MemoryNodeStore(INITIAL_CONTENT);
+ Oak oak = new Oak(nodeStore)
+ .with((QueryIndexProvider) queryIndexProvider)
+ .with((Observer) queryIndexProvider)
+ .with(editorProvider);
+
+ Jcr jcr = new Jcr(oak);
+ @NotNull Repository repository = jcr.createRepository();
+
+ session = repository.login(new SimpleCredentials("admin",
"admin".toCharArray()), null);
+ closer.register(session::logout);
+
+ // we'd always query anonymously
+ Session anonSession = repository.login(new GuestCredentials());
+ closer.register(anonSession::logout);
+ qe = anonSession.getWorkspace().getQueryManager();
+ }
+
+ @After
+ public void after() throws IOException {
+ closer.close();
+ IndexDefinition.setDisableStoredIndexDefinition(false);
+ }
+
+ private void createIndex() throws RepositoryException {
+ IndexDefinitionBuilder idxBuilder = new IndexDefinitionBuilder();
+ idxBuilder.noAsync().evaluatePathRestrictions()
+ .indexRule("nt:base")
+ .property("cons")
+ .propertyIndex()
+ .property("foo")
+ .getBuilderTree().setProperty(PROP_FACETS, true);
+
+ indexNode = getOrCreateByPath("/oak:index", "nt:unstructured", session)
+ .addNode("index", INDEX_DEFINITIONS_NODE_TYPE);
+ idxBuilder.build(indexNode);
+ session.save();
+ }
+
+ private void createLargeDataset() throws RepositoryException {
+ Random rGen = new Random(42);
+ int[] labelCount = new int[NUM_LABELS];
+ int[] aclLabelCount = new int[NUM_LABELS];
+ int[] aclPar1LabelCount = new int[NUM_LABELS];
+
+ Node par = allow(getOrCreateByPath("/parent", "oak:Unstructured",
session));
+
+ for (int i = 0; i < 4; i++) {
+ Node subPar = par.addNode("par" + i);
+ for (int j = 0; j < NUM_LEAF_NODES; j++) {
+ Node child = subPar.addNode("c" + j);
+ child.setProperty("cons", "val");
+
+ // Add a random label out of "l0", "l1", "l2", "l3"
+ int labelNum = rGen.nextInt(NUM_LABELS);
+ child.setProperty("foo", "l" + labelNum);
+
+ labelCount[labelNum]++;
+ if (i != 0) {
+ aclLabelCount[labelNum]++;
+ }
+ if (i == 1) {
+ aclPar1LabelCount[labelNum]++;
+ }
+ }
+
+ // deny access for one sub-parent
+ if (i == 0) {
+ deny(subPar);
+ }
+ }
+
+ session.save();
+
+ for (int i = 0; i < labelCount.length; i++) {
+ actualLabelCount.put("l" + i, labelCount[i]);
+ actualAclLabelCount.put("l" + i, aclLabelCount[i]);
+ actualAclPar1LabelCount.put("l" + i, aclPar1LabelCount[i]);
+ }
+
+ assertNotEquals("Acl-ed and actual counts mustn't be same",
actualLabelCount, actualAclLabelCount);
+ }
+
+ @Test
+ public void secureFacets() throws Exception {
+ createLargeDataset();
+
+ assertEquals(actualAclLabelCount, getFacets());
+ }
+
+ @Test
+ public void secureFacets_withOneLabelInaccessible() throws Exception {
+ createLargeDataset();
+ Node inaccessibleChild =
deny(session.getNode("/parent").addNode("par4")).addNode("c0");
+ inaccessibleChild.setProperty("cons", "val");
+ inaccessibleChild.setProperty("foo", "l4");
+ session.save();
+
+ assertEquals(actualAclLabelCount, getFacets());
+ }
+
+ @Test
+ public void insecureFacets() throws Exception {
+ Node facetConfig = getOrCreateByPath(indexNode.getPath() + "/" +
FACETS, "nt:unstructured", session);
+ facetConfig.setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_INSECURE);
+ indexNode.setProperty(PROP_REFRESH_DEFN, true);
+ session.save();
+
+ createLargeDataset();
+
+ assertEquals(actualLabelCount, getFacets());
+ }
+
+ @Test
+ public void statisticalFacets() throws Exception {
+ Node facetConfig = getOrCreateByPath(indexNode.getPath() + "/" +
FACETS, "nt:unstructured", session);
+ facetConfig.setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_STATISTICAL);
+ indexNode.setProperty(PROP_REFRESH_DEFN, true);
+ session.save();
+
+ createLargeDataset();
+
+ Map<String, Integer> facets = getFacets();
+ assertEquals("Unexpected number of facets",
actualAclLabelCount.size(), facets.size());
+
+ for (Map.Entry<String, Integer> facet :
actualAclLabelCount.entrySet()) {
+ String facetLabel = facet.getKey();
+ int facetCount = facets.get(facetLabel);
+ float ratio = ((float)facetCount) / facet.getValue();
+ assertTrue("Facet count for label: " + facetLabel + " is outside
of 10% margin of error. " +
+ "Expected: " + facet.getValue() + "; Got: " +
facetCount + "; Ratio: " + ratio,
+ Math.abs(ratio - 1) < 0.1);
+ }
+ }
+
+ @Test
+ public void statisticalFacets_withHitCountSameAsSampleSize() throws
Exception {
+ Node facetConfig = getOrCreateByPath(indexNode.getPath() + "/" +
FACETS, "nt:unstructured", session);
+ facetConfig.setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_STATISTICAL);
+ indexNode.setProperty(PROP_REFRESH_DEFN, true);
+ session.save();
+
+ createLargeDataset();
+
+ Map<String, Integer> facets = getFacets("/parent/par1");
+ assertEquals("Unexpected number of facets",
actualAclPar1LabelCount.size(), facets.size());
+
+ for (Map.Entry<String, Integer> facet :
actualAclPar1LabelCount.entrySet()) {
+ String facetLabel = facet.getKey();
+ int facetCount = facets.get(facetLabel);
+ float ratio = ((float)facetCount) / facet.getValue();
+ assertTrue("Facet count for label: " + facetLabel + " is outside
of 10% margin of error. " +
+ "Expected: " + facet.getValue() + "; Got: " +
facetCount + "; Ratio: " + ratio,
+ Math.abs(ratio - 1) < 0.1);
+ }
+ }
+
+ @Test
+ public void statisticalFacets_withOneLabelInaccessible() throws Exception {
+ Node facetConfig = getOrCreateByPath(indexNode.getPath() + "/" +
FACETS, "nt:unstructured", session);
+ facetConfig.setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_STATISTICAL);
+ indexNode.setProperty(PROP_REFRESH_DEFN, true);
+ session.save();
+
+ createLargeDataset();
+ Node inaccessibleChild =
deny(session.getNode("/parent").addNode("par4")).addNode("c0");
+ inaccessibleChild.setProperty("cons", "val");
+ inaccessibleChild.setProperty("foo", "l4");
+ session.save();
+
+ Map<String, Integer> facets = getFacets();
+ assertEquals("Unexpected number of facets",
actualAclLabelCount.size(), facets.size());
+
+ for (Map.Entry<String, Integer> facet :
actualAclLabelCount.entrySet()) {
+ String facetLabel = facet.getKey();
+ int facetCount = facets.get(facetLabel);
+ float ratio = ((float)facetCount) / facet.getValue();
+ assertTrue("Facet count for label: " + facetLabel + " is outside
of 10% margin of error. " +
+ "Expected: " + facet.getValue() + "; Got: " +
facetCount + "; Ratio: " + ratio,
+ Math.abs(ratio - 1) < 0.1);
+ }
+ }
+
+ @Test
+ public void secureFacets_withAdminSession() throws Exception {
+ Node facetConfig = getOrCreateByPath(indexNode.getPath() + "/" +
FACETS, "nt:unstructured", session);
+ facetConfig.setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_INSECURE);
+ indexNode.setProperty(PROP_REFRESH_DEFN, true);
+ session.save();
+
+ createLargeDataset();
+
+ qe = session.getWorkspace().getQueryManager();
+
+ assertEquals(actualLabelCount, getFacets());
+ }
+
+ @Test
+ public void statisticalFacets_withAdminSession() throws Exception {
+ Node facetConfig = getOrCreateByPath(indexNode.getPath() + "/" +
FACETS, "nt:unstructured", session);
+ facetConfig.setProperty(PROP_SECURE_FACETS,
PROP_SECURE_FACETS_VALUE_STATISTICAL);
+ indexNode.setProperty(PROP_REFRESH_DEFN, true);
+ session.save();
+
+ createLargeDataset();
+
+ qe = session.getWorkspace().getQueryManager();
+
+ Map<String, Integer> facets = getFacets();
+ assertEquals("Unexpected number of facets", actualLabelCount.size(),
facets.size());
+
+ for (Map.Entry<String, Integer> facet : actualLabelCount.entrySet()) {
+ String facetLabel = facet.getKey();
+ int facetCount = facets.get(facetLabel);
+ float ratio = ((float)facetCount) / facet.getValue();
+ assertTrue("Facet count for label: " + facetLabel + " is outside
of 5% margin of error. " +
+ "Expected: " + facet.getValue() + "; Got: " +
facetCount + "; Ratio: " + ratio,
+ Math.abs(ratio - 1) < 0.05);
+ }
+ }
+
+ private Map<String, Integer> getFacets() throws Exception {
+ return getFacets(null);
+ }
+
+ private Map<String, Integer> getFacets(String path) throws Exception {
+ Map<String, Integer> map = Maps.newHashMap();
+
+ String pathCons = "";
+ if (path != null) {
+ pathCons = " AND ISDESCENDANTNODE('" + path + "')";
+ }
+ String query = "SELECT [rep:facet(foo)] FROM [nt:base] WHERE [cons] =
'val'" + pathCons;
+
+ Query q = qe.createQuery(query, Query.JCR_SQL2);
+ QueryResult queryResult = q.execute();
+
+ FacetResult facetResult = new FacetResult(queryResult);
+
+ Set<String> dims = facetResult.getDimensions();
+ for (String dim : dims) {
+ List<Facet> facets = facetResult.getFacets(dim);
+ for (Facet facet : facets) {
+ map.put(facet.getLabel(), facet.getCount());
+ }
+ }
+
+ return map;
+ }
+
+ @SuppressWarnings("UnusedReturnValue")
+ private Node deny(Node node) throws RepositoryException {
+ AccessControlUtils.deny(node, "anonymous", Privilege.JCR_ALL);
+ return node;
+ }
+
+ private Node allow(Node node) throws RepositoryException {
+ AccessControlUtils.allow(node, "anonymous", Privilege.JCR_READ);
+ return node;
+ }
+}
Propchange:
jackrabbit/oak/branches/1.8/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/SecureFacetTest.java
------------------------------------------------------------------------------
svn:eol-style = native
