Author: reschke Date: Mon Oct 7 16:24:20 2019 New Revision: 1868086 URL: http://svn.apache.org/viewvc?rev=1868086&view=rev Log: OAK-8666: deprecate FileIOUtils.BurnOnCloseFileIterator
Added: jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/BurnOnCloseFileIterator.java (with props) Modified: jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java Modified: jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java?rev=1868086&r1=1868085&r2=1868086&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java (original) +++ jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java Mon Oct 7 16:24:20 2019 @@ -37,6 +37,7 @@ import com.google.common.io.Files; import org.apache.jackrabbit.core.data.DataRecord; import org.apache.jackrabbit.oak.commons.FileIOUtils.FileLineDifferenceIterator; import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser; +import org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator; import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore; import org.apache.jackrabbit.oak.stats.Clock; import org.jetbrains.annotations.Nullable; @@ -66,7 +67,6 @@ import static org.apache.commons.io.File import static org.apache.commons.io.FilenameUtils.concat; import static org.apache.commons.io.FilenameUtils.removeExtension; import static org.apache.commons.io.IOUtils.closeQuietly; -import static org.apache.jackrabbit.oak.commons.FileIOUtils.BurnOnCloseFileIterator.wrap; import static org.apache.jackrabbit.oak.commons.FileIOUtils.append; import static org.apache.jackrabbit.oak.commons.FileIOUtils.copy; import static org.apache.jackrabbit.oak.commons.FileIOUtils.sort; @@ -541,7 +541,7 @@ public class BlobIdTracker implements Cl try { // Get a temp file path String path = createTempFile("temp", null).getAbsolutePath(); - return wrap(lineIterator(getRecords(path)), new File(path)); + return BurnOnCloseFileIterator.wrap(lineIterator(getRecords(path)), new File(path)); } catch (IOException e) { LOG.error("Error in retrieving blob records iterator", e); throw e; Modified: jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java?rev=1868086&r1=1868085&r2=1868086&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java (original) +++ jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java Mon Oct 7 16:24:20 2019 @@ -456,21 +456,22 @@ public final class FileIOUtils { * A custom transformer can also be provided to unescape. * * @param <T> the type of elements in the iterator + * @deprecated use {@link org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator} instead */ - public static class BurnOnCloseFileIterator<T> extends AbstractIterator<T> implements Closeable { + @Deprecated public static class BurnOnCloseFileIterator<T> extends AbstractIterator<T> implements Closeable { private final Logger log = LoggerFactory.getLogger(getClass()); private final LineIterator iterator; private final Function<String, T> transformer; - private File backingFile; + private final File backingFile; public BurnOnCloseFileIterator(LineIterator iterator, Function<String, T> transformer) { - this.iterator = iterator; - this.transformer = transformer; + this(iterator, null, transformer); } public BurnOnCloseFileIterator(LineIterator iterator, File backingFile, Function<String, T> transformer) { + GuavaDeprecation.handleCall("OAK-8666"); this.iterator = iterator; this.transformer = transformer; this.backingFile = backingFile; Added: jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/BurnOnCloseFileIterator.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/BurnOnCloseFileIterator.java?rev=1868086&view=auto ============================================================================== --- jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/BurnOnCloseFileIterator.java (added) +++ jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/BurnOnCloseFileIterator.java Mon Oct 7 16:24:20 2019 @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.commons.io; + +import static org.apache.commons.io.FileUtils.forceDelete; +import static org.apache.jackrabbit.oak.commons.IOUtils.closeQuietly; + +import java.io.BufferedWriter; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.util.Iterator; +import java.util.function.Function; + +import org.apache.commons.io.LineIterator; +import org.apache.jackrabbit.oak.commons.FileIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.AbstractIterator; + +/** + * Implements a {@link java.io.Closeable} wrapper over a {@link LineIterator}. + * Also has a transformer to transform the output. If the underlying file is + * provided then it deletes the file on {@link #close()}. + * + * If there is a scope for lines in the file containing line break characters it + * should be ensured that the files is written with + * {@link FileIOUtils#writeAsLine(BufferedWriter, String, boolean)} with true to escape + * line break characters and should be properly unescaped on read. A custom + * transformer can also be provided to unescape. + * + * @param <T> + * the type of elements in the iterator + */ +public class BurnOnCloseFileIterator<T> implements Closeable, Iterator<T> { + + private static final Logger LOG = LoggerFactory.getLogger(BurnOnCloseFileIterator.class); + + private final Impl<T> delegate; + + public BurnOnCloseFileIterator(Iterator<String> iterator, Function<String, T> transformer) { + this.delegate = new Impl<T>(iterator, null, transformer); + } + + public BurnOnCloseFileIterator(Iterator<String> iterator, File backingFile, Function<String, T> transformer) { + this.delegate = new Impl<T>(iterator, backingFile, transformer); + } + + @Override + public boolean hasNext() { + return this.delegate.hasNext(); + } + + @Override + public T next() { + return this.delegate.next(); + } + + @Override + public void close() throws IOException { + this.delegate.close(); + } + + public static BurnOnCloseFileIterator<String> wrap(Iterator<String> iter) { + return new BurnOnCloseFileIterator<String>(iter, new Function<String, String>() { + public String apply(String s) { + return s; + } + }); + } + + public static BurnOnCloseFileIterator<String> wrap(Iterator<String> iter, File backingFile) { + return new BurnOnCloseFileIterator<String>(iter, backingFile, new Function<String, String>() { + public String apply(String s) { + return s; + } + }); + } + + private static class Impl<T> extends AbstractIterator<T> implements Closeable { + private final Iterator<String> iterator; + private final Function<String, T> transformer; + private final File backingFile; + + public Impl(Iterator<String> iterator, File backingFile, Function<String, T> transformer) { + this.iterator = iterator; + this.transformer = transformer; + this.backingFile = backingFile; + } + + @Override + protected T computeNext() { + if (iterator.hasNext()) { + return transformer.apply(iterator.next()); + } + + try { + close(); + } catch (IOException e) { + LOG.warn("Error closing iterator", e); + } + return endOfData(); + } + + @Override + public void close() throws IOException { + if (iterator instanceof Closeable) { + closeQuietly((Closeable) iterator); + } + if (backingFile != null && backingFile.exists()) { + forceDelete(backingFile); + } + } + } +} Propchange: jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/io/BurnOnCloseFileIterator.java ------------------------------------------------------------------------------ svn:eol-style = native Modified: jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java?rev=1868086&r1=1868085&r2=1868086&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java (original) +++ jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java Mon Oct 7 16:24:20 2019 @@ -338,7 +338,19 @@ public class FileIOUtilsTest { Set<String> added = newHashSet("a", "z", "e", "b"); File f = assertWrite(added.iterator(), false, added.size()); - BurnOnCloseFileIterator iterator = + org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator<String> iterator = + org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator.wrap(FileUtils.lineIterator(f, UTF_8.toString())); + + assertEquals(added, Sets.newHashSet(iterator)); + assertTrue(f.exists()); + } + + @Test + public void deprecatedfileIteratorTest() throws Exception { + Set<String> added = newHashSet("a", "z", "e", "b"); + File f = assertWrite(added.iterator(), false, added.size()); + + BurnOnCloseFileIterator<String> iterator = BurnOnCloseFileIterator.wrap(FileUtils.lineIterator(f, UTF_8.toString())); assertEquals(added, Sets.newHashSet(iterator)); @@ -350,7 +362,19 @@ public class FileIOUtilsTest { Set<String> added = newHashSet("a", "z", "e", "b"); File f = assertWrite(added.iterator(), false, added.size()); - BurnOnCloseFileIterator iterator = + org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator<String> iterator = + org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator.wrap(FileUtils.lineIterator(f, UTF_8.toString()), f); + + assertEquals(added, Sets.newHashSet(iterator)); + assertTrue(!f.exists()); + } + + @Test + public void deprecatedFileIteratorBurnTest() throws Exception { + Set<String> added = newHashSet("a", "z", "e", "b"); + File f = assertWrite(added.iterator(), false, added.size()); + + BurnOnCloseFileIterator<String> iterator = BurnOnCloseFileIterator.wrap(FileUtils.lineIterator(f, UTF_8.toString()), f); assertEquals(added, Sets.newHashSet(iterator)); @@ -362,13 +386,20 @@ public class FileIOUtilsTest { Set<String> added = newHashSet(getLineBreakStrings()); File f = assertWrite(added.iterator(), true, added.size()); - BurnOnCloseFileIterator iterator = - new BurnOnCloseFileIterator<String>(FileUtils.lineIterator(f, UTF_8.toString()), f, - new Function<String, String>() { - @Nullable @Override public String apply(@Nullable String input) { - return unescapeLineBreaks(input); - } - }); + org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator<String> iterator = new org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator<String>( + FileUtils.lineIterator(f, UTF_8.toString()), f, (input) -> unescapeLineBreaks(input)); + + assertEquals(added, Sets.newHashSet(iterator)); + assertTrue(!f.exists()); + } + + @Test + public void deprecatedFileIteratorLineBreakTest() throws IOException { + Set<String> added = newHashSet(getLineBreakStrings()); + File f = assertWrite(added.iterator(), true, added.size()); + + BurnOnCloseFileIterator<String> iterator = new BurnOnCloseFileIterator<String>(FileUtils.lineIterator(f, UTF_8.toString()), + f, (input) -> unescapeLineBreaks(input)); assertEquals(added, Sets.newHashSet(iterator)); assertTrue(!f.exists()); @@ -382,16 +413,23 @@ public class FileIOUtilsTest { } File f = assertWrite(added.iterator(), true, added.size()); - BurnOnCloseFileIterator iterator = - new BurnOnCloseFileIterator<String>(FileUtils.lineIterator(f, UTF_8.toString()), - f, - new Function<String, String>() { - @Nullable - @Override - public String apply(@Nullable String input) { - return unescapeLineBreaks(input); - } - }); + org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator<String> iterator = new org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator<String>( + FileUtils.lineIterator(f, UTF_8.toString()), f, (input) -> unescapeLineBreaks(input)); + + assertEquals(added, Sets.newHashSet(iterator)); + assertTrue(!f.exists()); + } + + @Test + public void deprecatedFileIteratorRandomizedTest() throws Exception { + Set<String> added = newHashSet(); + for (int i = 0; i < 100; i++) { + added.add(getRandomTestString()); + } + File f = assertWrite(added.iterator(), true, added.size()); + + BurnOnCloseFileIterator<String> iterator = new BurnOnCloseFileIterator<String>(FileUtils.lineIterator(f, UTF_8.toString()), + f, (input) -> unescapeLineBreaks(input)); assertEquals(added, Sets.newHashSet(iterator)); assertTrue(!f.exists()); Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java?rev=1868086&r1=1868085&r2=1868086&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java Mon Oct 7 16:24:20 2019 @@ -24,8 +24,8 @@ import java.util.Iterator; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.function.Function; -import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.io.Closer; @@ -36,9 +36,9 @@ import org.apache.jackrabbit.oak.api.Blo import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.commons.FileIOUtils; -import org.apache.jackrabbit.oak.commons.FileIOUtils.BurnOnCloseFileIterator; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser; +import org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator; import org.apache.jackrabbit.oak.commons.sort.EscapeUtils; import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever; import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector; @@ -336,8 +336,8 @@ public class DataStoreCommand implements public void log() throws IOException { File tempFile = new File(outDir, outFile.getName() + "-temp"); FileUtils.moveFile(outFile, tempFile); - try (BurnOnCloseFileIterator iterator = - new BurnOnCloseFileIterator(FileUtils.lineIterator(tempFile, UTF_8.toString()), tempFile, + try (BurnOnCloseFileIterator<String> iterator = + new BurnOnCloseFileIterator<String>(FileUtils.lineIterator(tempFile, UTF_8.toString()), tempFile, (Function<String, String>) input -> encodeId(input, blobStoreType))) { FileIOUtils.writeStrings(iterator, outFile, true, log, "Transformed to verbose ids - "); }