Author: chetanm Date: Fri Dec 15 09:38:21 2017 New Revision: 1818252 URL: http://svn.apache.org/viewvc?rev=1818252&view=rev Log: OAK-6353 - Use Document order traversal for reindexing performed on DocumentNodeStore setups
Reconfigure the progress reporter and also track the entryCount in FlatFileStore Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java?rev=1818252&r1=1818251&r2=1818252&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java Fri Dec 15 09:38:21 2017 @@ -63,7 +63,7 @@ public class DocumentStoreIndexer implem private final IndexHelper indexHelper; private final List<NodeStateIndexerProvider> indexerProviders; private final IndexerSupport indexerSupport; - private final IndexingProgressReporter progressReporter = + private IndexingProgressReporter progressReporter = new IndexingProgressReporter(IndexUpdateCallback.NOOP, NodeTraversalCallback.NOOP); public DocumentStoreIndexer(IndexHelper indexHelper, IndexerSupport indexerSupport) throws IOException { @@ -108,6 +108,8 @@ public class DocumentStoreIndexer implem .build(); closer.register(flatFileStore); + reconfigureReporter(flatFileStore); + progressReporter.reindexingTraversalStart("/"); for (NodeStateEntry entry : flatFileStore) { @@ -128,18 +130,41 @@ public class DocumentStoreIndexer implem } private void configureEstimators() { + configureTraversalRateEstimator(progressReporter); + long nodesCount = getEstimatedDocumentCount(); + if (nodesCount > 0) { + progressReporter.setNodeCountEstimator((String basePath, Set<String> indexPaths) -> nodesCount); + log.info("Estimated number of documents in Mongo are {}", nodesCount); + } + } + + private void reconfigureReporter(FlatFileStore flatFileStore) { + progressReporter = + new IndexingProgressReporter(IndexUpdateCallback.NOOP, NodeTraversalCallback.NOOP); + configureTraversalRateEstimator(progressReporter); + long entryCount = flatFileStore.getEntryCount(); + if (entryCount > 0) { + progressReporter.setNodeCountEstimator((String basePath, Set<String> indexPaths) -> entryCount); + log.info("Estimated number of entries in flat file store are {}", entryCount); + } else { + log.info("Number of entries in flat file store are unknown"); + } + } + + private void configureTraversalRateEstimator(IndexingProgressReporter progressReporter) { StatisticsProvider statsProvider = indexHelper.getStatisticsProvider(); if (statsProvider instanceof MetricStatisticsProvider) { MetricRegistry registry = ((MetricStatisticsProvider) statsProvider).getRegistry(); progressReporter.setTraversalRateEstimator(new MetricRateEstimator("async", registry)); } + } + private long getEstimatedDocumentCount(){ MongoConnection mongoConnection = indexHelper.getService(MongoConnection.class); if (mongoConnection != null) { - long nodesCount = mongoConnection.getDB().getCollection("nodes").count(); - progressReporter.setNodeCountEstimator((String basePath, Set<String> indexPaths) -> nodesCount); - log.info("Estimated number of documents in Mongo are {}", nodesCount); + return mongoConnection.getDB().getCollection("nodes").count(); } + return 0; } @Override Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java?rev=1818252&r1=1818251&r2=1818252&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java Fri Dec 15 09:38:21 2017 @@ -46,6 +46,7 @@ public class FlatFileNodeStoreBuilder { private final File workDir; private Iterable<String> preferredPathElements = Collections.emptySet(); private BlobStore blobStore; + private long entryCount = 0; private boolean useZip = Boolean.getBoolean(OAK_INDEXER_USE_ZIP); private boolean deleteOriginal = Boolean.parseBoolean(System.getProperty(OAK_INDEXER_DELETE_ORIGINAL, "true")); @@ -68,7 +69,11 @@ public class FlatFileNodeStoreBuilder { public FlatFileStore build() throws IOException { //TODO Check not null blobStore - return new FlatFileStore(createdSortedStoreFile(), new NodeStateEntryReader(blobStore), size(preferredPathElements)); + FlatFileStore store = new FlatFileStore(createdSortedStoreFile(), new NodeStateEntryReader(blobStore), size(preferredPathElements)); + if (entryCount > 0) { + store.setEntryCount(entryCount); + } + return store; } private File createdSortedStoreFile() throws IOException { @@ -121,9 +126,10 @@ public class FlatFileNodeStoreBuilder { ) { for (NodeStateEntry e : nodeStates) { entryWriter.write(e); + entryCount++; } } - log.info("Dumped nodestates in json format in {}", sw); + log.info("Dumped {} nodestates in json format in {}",entryCount, sw); return file; } Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java?rev=1818252&r1=1818251&r2=1818252&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java Fri Dec 15 09:38:21 2017 @@ -38,6 +38,7 @@ public class FlatFileStore implements It private final File storeFile; private final NodeStateEntryReader entryReader; private final int checkChildLimit; + private long entryCount = -1; public FlatFileStore(File storeFile, NodeStateEntryReader entryReader, int checkChildLimit) { this.storeFile = storeFile; @@ -45,6 +46,14 @@ public class FlatFileStore implements It this.checkChildLimit = checkChildLimit; } + public long getEntryCount() { + return entryCount; + } + + public void setEntryCount(long entryCount) { + this.entryCount = entryCount; + } + @Override public Iterator<NodeStateEntry> iterator() { return new FlatFileStoreIterator(createBaseIterator(), checkChildLimit);