Author: chetanm Date: Thu Dec 21 04:45:05 2017 New Revision: 1818876 URL: http://svn.apache.org/viewvc?rev=1818876&view=rev Log: OAK-7102 - Refactor DocumentIndexer logic to enable different sort approaches
Refactor to centralize the reader and writer creation with potential support for compression (set to false for now) Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreUtils.java (with props) Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java?rev=1818876&r1=1818875&r2=1818876&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java Thu Dec 21 04:45:05 2017 @@ -19,15 +19,13 @@ package org.apache.jackrabbit.oak.index.indexer.document.flatfile; +import java.io.BufferedWriter; import java.io.File; import java.io.IOException; -import java.io.Writer; import java.util.Collections; -import com.google.common.base.Charsets; import com.google.common.base.Stopwatch; import com.google.common.collect.Iterables; -import com.google.common.io.Files; import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.commons.IOUtils; import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; @@ -71,7 +69,8 @@ public class FlatFileNodeStoreBuilder { public FlatFileStore build() throws IOException { log.info("Preferred path elements are {}", Iterables.toString(preferredPathElements)); - FlatFileStore store = new FlatFileStore(createdSortedStoreFile(), new NodeStateEntryReader(blobStore), size(preferredPathElements)); + FlatFileStore store = new FlatFileStore(createdSortedStoreFile(), new NodeStateEntryReader(blobStore), + size(preferredPathElements), false); if (entryCount > 0) { store.setEntryCount(entryCount); } @@ -123,7 +122,7 @@ public class FlatFileNodeStoreBuilder { File file = new File(dir, fileName); Stopwatch sw = Stopwatch.createStarted(); NodeStateEntryWriter entryWriter = new NodeStateEntryWriter(blobStore); - try (Writer w = Files.newWriter(file, Charsets.UTF_8)) { + try (BufferedWriter w = FlatFileStoreUtils.createWriter(file, false)) { for (NodeStateEntry e : nodeStates) { String line = entryWriter.toString(e); w.append(line).append(LINE_SEPARATOR.value()); Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java?rev=1818876&r1=1818875&r2=1818876&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java Thu Dec 21 04:45:05 2017 @@ -21,29 +21,29 @@ package org.apache.jackrabbit.oak.index. import java.io.Closeable; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; -import java.io.Reader; import java.util.Iterator; -import com.google.common.base.Charsets; import com.google.common.collect.AbstractIterator; import com.google.common.io.Closer; -import com.google.common.io.Files; import org.apache.commons.io.LineIterator; import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileStoreUtils.createReader; + public class FlatFileStore implements Iterable<NodeStateEntry>, Closeable{ private final Closer closer = Closer.create(); private final File storeFile; private final NodeStateEntryReader entryReader; private final int checkChildLimit; + private final boolean compressionEnabled; private long entryCount = -1; - public FlatFileStore(File storeFile, NodeStateEntryReader entryReader, int checkChildLimit) { + public FlatFileStore(File storeFile, NodeStateEntryReader entryReader, int checkChildLimit, boolean compressionEnabled) { this.storeFile = storeFile; this.entryReader = entryReader; this.checkChildLimit = checkChildLimit; + this.compressionEnabled = compressionEnabled; } public long getEntryCount() { @@ -60,7 +60,7 @@ public class FlatFileStore implements It } private Iterator<NodeStateEntry> createBaseIterator() { - LineIterator itr = new LineIterator(createReader()); + LineIterator itr = new LineIterator(createReader(storeFile, compressionEnabled)); closer.register(itr::close); return new AbstractIterator<NodeStateEntry>() { @Override @@ -84,12 +84,4 @@ public class FlatFileStore implements It public void close() throws IOException { closer.close(); } - - private Reader createReader() { - try { - return Files.newReader(storeFile, Charsets.UTF_8); - } catch (FileNotFoundException e) { - throw new RuntimeException("Error opening file " + storeFile, e); - } - } } Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreUtils.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreUtils.java?rev=1818876&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreUtils.java (added) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreUtils.java Thu Dec 21 04:45:05 2017 @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.index.indexer.document.flatfile; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.util.zip.Deflater; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import static com.google.common.base.Charsets.UTF_8; + +class FlatFileStoreUtils { + + public static BufferedReader createReader(File file, boolean compressionEnabled) { + try { + BufferedReader br; + InputStream in = new FileInputStream(file); + if (compressionEnabled) { + br = new BufferedReader(new InputStreamReader(new GZIPInputStream(in, 2048), UTF_8)); + } else { + br = new BufferedReader(new InputStreamReader(in, UTF_8)); + } + return br; + } catch (IOException e) { + throw new RuntimeException("Error opening file " + file, e); + } + } + + public static BufferedWriter createWriter(File file, boolean compressionEnabled) throws IOException { + OutputStream out = new FileOutputStream(file); + if (compressionEnabled) { + out = new GZIPOutputStream(out, 2048) { + { + def.setLevel(Deflater.BEST_SPEED); + } + }; + } + return new BufferedWriter(new OutputStreamWriter(out, UTF_8)); + } +} Propchange: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreUtils.java ------------------------------------------------------------------------------ svn:eol-style = native