Author: chetanm Date: Fri Dec 8 07:26:25 2017 New Revision: 1817464 URL: http://svn.apache.org/viewvc?rev=1817464&view=rev Log: OAK-6353 - Use Document order traversal for reindexing performed on DocumentNodeStore setups
Make sorting tuning parameters configurable via system property with following defaults - Compression NOT enabled - Max memory to be used for sorting set to 5GB - Original file would be deleted before merge phase Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java?rev=1817464&r1=1817463&r2=1817464&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java Fri Dec 8 07:26:25 2017 @@ -29,13 +29,23 @@ import com.google.common.io.Files; import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FlatFileNodeStoreBuilder { + private static final String OAK_INDEXER_USE_ZIP = "oak.indexer.useZip"; + private static final String OAK_INDEXER_DELETE_ORIGINAL = "oak.indexer.deleteOriginal"; + private static final String OAK_INDEXER_MAX_SORT_MEMORY_IN_GB = "oak.indexer.maxSortMemoryInGB"; + private final Logger log = LoggerFactory.getLogger(getClass()); private final Iterable<NodeStateEntry> nodeStates; private final File workDir; private Iterable<String> preferredPathElements = Collections.emptySet(); private BlobStore blobStore; + private boolean useZip = Boolean.getBoolean(OAK_INDEXER_USE_ZIP); + private boolean deleteOriginal = Boolean.parseBoolean(System.getProperty(OAK_INDEXER_DELETE_ORIGINAL, "true")); + private int maxMemory = Integer.getInteger(OAK_INDEXER_MAX_SORT_MEMORY_IN_GB, 5); + public FlatFileNodeStoreBuilder(Iterable<NodeStateEntry> nodeStates, File workDir) { this.nodeStates = nodeStates; this.workDir = workDir; @@ -64,11 +74,22 @@ public class FlatFileNodeStoreBuilder { FileUtils.forceMkdir(sortWorkDir); NodeStateEntrySorter sorter = new NodeStateEntrySorter(new PathElementComparator(preferredPathElements), storeFile, sortWorkDir); - //TODO Configure flags zip and deleteOriginal, maxMemory + + logFlags(); + + sorter.setUseZip(useZip); + sorter.setMaxMemoryInGB(maxMemory); + sorter.setDeleteOriginal(deleteOriginal); sorter.sort(); return sorter.getSortedFile(); } + private void logFlags() { + log.info("Compression enabled while sorting : {} ({})", useZip, OAK_INDEXER_USE_ZIP); + log.info("Delete original dump from traversal : {} ({})", deleteOriginal, OAK_INDEXER_DELETE_ORIGINAL); + log.info("Max heap memory (GB) to be used for merge sort : {} ({})", maxMemory, OAK_INDEXER_MAX_SORT_MEMORY_IN_GB); + } + private File writeToStore(File dir, String fileName) throws IOException { File file = new File(dir, fileName); try (