Author: chetanm
Date: Fri Dec 15 09:38:21 2017
New Revision: 1818252
URL: http://svn.apache.org/viewvc?rev=1818252&view=rev
Log:
OAK-6353 - Use Document order traversal for reindexing performed on
DocumentNodeStore setups
Reconfigure the progress reporter and also track the entryCount in FlatFileStore
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java?rev=1818252&r1=1818251&r2=1818252&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
Fri Dec 15 09:38:21 2017
@@ -63,7 +63,7 @@ public class DocumentStoreIndexer implem
private final IndexHelper indexHelper;
private final List<NodeStateIndexerProvider> indexerProviders;
private final IndexerSupport indexerSupport;
- private final IndexingProgressReporter progressReporter =
+ private IndexingProgressReporter progressReporter =
new IndexingProgressReporter(IndexUpdateCallback.NOOP,
NodeTraversalCallback.NOOP);
public DocumentStoreIndexer(IndexHelper indexHelper, IndexerSupport
indexerSupport) throws IOException {
@@ -108,6 +108,8 @@ public class DocumentStoreIndexer implem
.build();
closer.register(flatFileStore);
+ reconfigureReporter(flatFileStore);
+
progressReporter.reindexingTraversalStart("/");
for (NodeStateEntry entry : flatFileStore) {
@@ -128,18 +130,41 @@ public class DocumentStoreIndexer implem
}
private void configureEstimators() {
+ configureTraversalRateEstimator(progressReporter);
+ long nodesCount = getEstimatedDocumentCount();
+ if (nodesCount > 0) {
+ progressReporter.setNodeCountEstimator((String basePath,
Set<String> indexPaths) -> nodesCount);
+ log.info("Estimated number of documents in Mongo are {}",
nodesCount);
+ }
+ }
+
+ private void reconfigureReporter(FlatFileStore flatFileStore) {
+ progressReporter =
+ new IndexingProgressReporter(IndexUpdateCallback.NOOP,
NodeTraversalCallback.NOOP);
+ configureTraversalRateEstimator(progressReporter);
+ long entryCount = flatFileStore.getEntryCount();
+ if (entryCount > 0) {
+ progressReporter.setNodeCountEstimator((String basePath,
Set<String> indexPaths) -> entryCount);
+ log.info("Estimated number of entries in flat file store are {}",
entryCount);
+ } else {
+ log.info("Number of entries in flat file store are unknown");
+ }
+ }
+
+ private void configureTraversalRateEstimator(IndexingProgressReporter
progressReporter) {
StatisticsProvider statsProvider = indexHelper.getStatisticsProvider();
if (statsProvider instanceof MetricStatisticsProvider) {
MetricRegistry registry = ((MetricStatisticsProvider)
statsProvider).getRegistry();
progressReporter.setTraversalRateEstimator(new
MetricRateEstimator("async", registry));
}
+ }
+ private long getEstimatedDocumentCount(){
MongoConnection mongoConnection =
indexHelper.getService(MongoConnection.class);
if (mongoConnection != null) {
- long nodesCount =
mongoConnection.getDB().getCollection("nodes").count();
- progressReporter.setNodeCountEstimator((String basePath,
Set<String> indexPaths) -> nodesCount);
- log.info("Estimated number of documents in Mongo are {}",
nodesCount);
+ return mongoConnection.getDB().getCollection("nodes").count();
}
+ return 0;
}
@Override
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java?rev=1818252&r1=1818251&r2=1818252&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
Fri Dec 15 09:38:21 2017
@@ -46,6 +46,7 @@ public class FlatFileNodeStoreBuilder {
private final File workDir;
private Iterable<String> preferredPathElements = Collections.emptySet();
private BlobStore blobStore;
+ private long entryCount = 0;
private boolean useZip = Boolean.getBoolean(OAK_INDEXER_USE_ZIP);
private boolean deleteOriginal =
Boolean.parseBoolean(System.getProperty(OAK_INDEXER_DELETE_ORIGINAL, "true"));
@@ -68,7 +69,11 @@ public class FlatFileNodeStoreBuilder {
public FlatFileStore build() throws IOException {
//TODO Check not null blobStore
- return new FlatFileStore(createdSortedStoreFile(), new
NodeStateEntryReader(blobStore), size(preferredPathElements));
+ FlatFileStore store = new FlatFileStore(createdSortedStoreFile(), new
NodeStateEntryReader(blobStore), size(preferredPathElements));
+ if (entryCount > 0) {
+ store.setEntryCount(entryCount);
+ }
+ return store;
}
private File createdSortedStoreFile() throws IOException {
@@ -121,9 +126,10 @@ public class FlatFileNodeStoreBuilder {
) {
for (NodeStateEntry e : nodeStates) {
entryWriter.write(e);
+ entryCount++;
}
}
- log.info("Dumped nodestates in json format in {}", sw);
+ log.info("Dumped {} nodestates in json format in {}",entryCount, sw);
return file;
}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java?rev=1818252&r1=1818251&r2=1818252&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
Fri Dec 15 09:38:21 2017
@@ -38,6 +38,7 @@ public class FlatFileStore implements It
private final File storeFile;
private final NodeStateEntryReader entryReader;
private final int checkChildLimit;
+ private long entryCount = -1;
public FlatFileStore(File storeFile, NodeStateEntryReader entryReader, int
checkChildLimit) {
this.storeFile = storeFile;
@@ -45,6 +46,14 @@ public class FlatFileStore implements It
this.checkChildLimit = checkChildLimit;
}
+ public long getEntryCount() {
+ return entryCount;
+ }
+
+ public void setEntryCount(long entryCount) {
+ this.entryCount = entryCount;
+ }
+
@Override
public Iterator<NodeStateEntry> iterator() {
return new FlatFileStoreIterator(createBaseIterator(),
checkChildLimit);
