NIFI-527: Compress prov logs in 'chunks' and just store the chunk offsets in Lucene instead of byte offsets
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/7c41225e Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/7c41225e Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/7c41225e Branch: refs/heads/develop Commit: 7c41225e89e05fe1a234920fd1aa8ca248f43850 Parents: a5ac48a Author: Mark Payne <[email protected]> Authored: Mon Apr 27 10:41:18 2015 -0400 Committer: Mark Payne <[email protected]> Committed: Mon Apr 27 10:41:18 2015 -0400 ---------------------------------------------------------------------- .../nifi/stream/io/ByteCountingInputStream.java | 5 +++++ .../nifi/stream/io/ByteCountingOutputStream.java | 8 ++++++++ .../TestPersistentProvenanceRepository.java | 19 ++++++++++++++++++- 3 files changed, 31 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/7c41225e/nifi/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingInputStream.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingInputStream.java b/nifi/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingInputStream.java index 8294af3..d1ed023 100644 --- a/nifi/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingInputStream.java +++ b/nifi/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingInputStream.java @@ -31,6 +31,11 @@ public class ByteCountingInputStream extends InputStream { this.in = in; } + public ByteCountingInputStream(final InputStream in, final long initialOffset) { + this.in = in; + this.bytesSkipped = initialOffset; + } + @Override public int read() throws IOException { final int fromSuper = in.read(); http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/7c41225e/nifi/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingOutputStream.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingOutputStream.java b/nifi/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingOutputStream.java index 3e3e3fe..e71937e 100644 --- a/nifi/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingOutputStream.java +++ b/nifi/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingOutputStream.java @@ -27,6 +27,12 @@ public class ByteCountingOutputStream extends OutputStream { public ByteCountingOutputStream(final OutputStream out) { this.out = out; } + + public ByteCountingOutputStream(final OutputStream out, final long initialByteCount) { + this.out = out; + this.bytesWritten = initialByteCount; + } + @Override public void write(int b) throws IOException { @@ -39,6 +45,8 @@ public class ByteCountingOutputStream extends OutputStream { write(b, 0, b.length); } + ; + @Override public void write(byte[] b, int off, int len) throws IOException { out.write(b, off, len); http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/7c41225e/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java index 25a363f..5541ab5 100644 --- a/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java +++ b/nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java @@ -110,7 +110,24 @@ public class TestPersistentProvenanceRepository { // we create but also to ensure that we have closed all of the file handles. If we leave any // streams open, for instance, this will throw an IOException, causing our unit test to fail. for ( final File storageDir : config.getStorageDirectories() ) { - FileUtils.deleteFile(storageDir, true); + int i; + for (i=0; i < 3; i++) { + try { + FileUtils.deleteFile(storageDir, true); + break; + } catch (final IOException ioe) { + // if there is a virus scanner, etc. running in the background we may not be able to + // delete the file. Wait a sec and try again. + if ( i == 2 ) { + throw ioe; + } else { + try { + Thread.sleep(1000L); + } catch (final InterruptedException ie) { + } + } + } + } } }
