Author: toad
Date: 2008-11-29 20:04:11 +0000 (Sat, 29 Nov 2008)
New Revision: 23986
Modified:
branches/db4o/freenet/
branches/db4o/freenet/src/freenet/client/FECCodec.java
branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSegment.java
branches/db4o/freenet/src/freenet/clients/http/filter/PNGFilter.java
branches/db4o/freenet/src/freenet/crypt/Yarrow.java
branches/db4o/freenet/src/freenet/node/Version.java
branches/db4o/freenet/src/freenet/support/BinaryBloomFilter.java
branches/db4o/freenet/src/freenet/support/BloomFilter.java
branches/db4o/freenet/src/freenet/support/CountingBloomFilter.java
branches/db4o/freenet/src/freenet/support/compress/Bzip2Compressor.java
branches/db4o/freenet/src/freenet/support/compress/GzipCompressor.java
branches/db4o/freenet/src/freenet/support/compress/LZMACompressor.java
branches/db4o/freenet/src/freenet/support/io/CountedInputStream.java
branches/db4o/freenet/test/freenet/crypt/YarrowTest.java
branches/db4o/freenet/test/freenet/support/FieldsTest.java
branches/db4o/freenet/test/freenet/support/io/MockInputStream.java
Log:
Merge 1182 (Last commit was actually merge up to 23794, should have been 23784
= 1181)
Property changes on: branches/db4o/freenet
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/freenet:19964-23794
+ /trunk/freenet:19964-23821
Modified: branches/db4o/freenet/src/freenet/client/FECCodec.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/FECCodec.java 2008-11-29
19:57:22 UTC (rev 23985)
+++ branches/db4o/freenet/src/freenet/client/FECCodec.java 2008-11-29
20:04:11 UTC (rev 23986)
@@ -128,6 +128,18 @@
packets[i] = new Buffer(realBuffer, i *
STRIPE_SIZE,
STRIPE_SIZE);
+ // Shortcut.
+ // Due to the not-fetching-last-block code, we need to
check here,
+ // rather than relying on numberToDecode (since the
last data block won't be part of numberToDecode).
+
+ boolean needDecode = false;
+ for(int i = 0; i < dataBlockStatus.length;i++) {
+ if(dataBlockStatus[i].getData() == null)
+ needDecode = true;
+ }
+
+ if(!needDecode) return;
+
for(int i = 0; i < dataBlockStatus.length; i++) {
buckets[i] = dataBlockStatus[i].getData();
if(buckets[i] == null) {
Modified:
branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSegment.java
===================================================================
--- branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSegment.java
2008-11-29 19:57:22 UTC (rev 23985)
+++ branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSegment.java
2008-11-29 20:04:11 UTC (rev 23986)
@@ -218,8 +218,8 @@
((ClientGetter)parent).addKeyToBinaryBlob(block,
container, context);
// No need to unregister key, because it will be cleared in
tripPendingKey().
boolean dontNotify;
- boolean haveDataBlocks;
boolean wasDataBlock = false;
+ boolean allFailed = false;
synchronized(this) {
if(finished) {
// Happens sometimes, don't complain about it...
@@ -290,16 +290,25 @@
// Don't count the last data block, since we
can't use it in FEC decoding.
if(!(ignoreLastDataBlock && blockNo ==
dataKeys.length - 1))
fetchedBlocks++;
+ else
+ // This block is not going to be
fetched, and because of the insertion format.
+ // Thus it is a fatal failure. We need
to track it, because it is quite possible
+ // to fetch the last block, not
complete because it's the last block, and hang.
+ fatallyFailedBlocks++;
// However, if we manage to get EVERY data
block (common on a small splitfile),
// we don't need to FEC decode.
if(wasDataBlock)
fetchedDataBlocks++;
if(logMINOR) Logger.minor(this, "Fetched
"+fetchedBlocks+" blocks in onSuccess("+blockNo+")");
- haveDataBlocks = fetchedDataBlocks ==
dataKeys.length;
+ boolean haveDataBlocks = fetchedDataBlocks ==
dataKeys.length;
decodeNow = (!startedDecode) && (fetchedBlocks
>= minFetched || haveDataBlocks);
if(decodeNow) {
startedDecode = true;
finishing = true;
+ } else {
+ // Avoid hanging when we have one n-1
check blocks, we succeed on the last data block,
+ // we don't have the other data blocks,
and we have nothing else fetching.
+ allFailed = failedBlocks +
fatallyFailedBlocks > (dataKeys.length + checkKeys.length - minFetched);
}
}
dontNotify = !scheduled;
@@ -316,10 +325,9 @@
if(persistent)
container.deactivate(parentFetcher, 1);
removeSubSegments(container, context);
- if(haveDataBlocks)
- onDecodedSegment(container, context, null,
null, null, dataBuckets, checkBuckets);
- else
- decode(container, context);
+ decode(container, context);
+ } else if(allFailed) {
+ fail(new FetchException(FetchException.SPLITFILE_ERROR,
errors), container, context, true);
}
if(persistent) {
container.deactivate(parent, 1);
Modified: branches/db4o/freenet/src/freenet/clients/http/filter/PNGFilter.java
===================================================================
--- branches/db4o/freenet/src/freenet/clients/http/filter/PNGFilter.java
2008-11-29 19:57:22 UTC (rev 23985)
+++ branches/db4o/freenet/src/freenet/clients/http/filter/PNGFilter.java
2008-11-29 20:04:11 UTC (rev 23986)
@@ -60,7 +60,7 @@
// http://wiki.mozilla.org/APNG_Specification
"acTL",
"fcTL",
- "fDAT"
+ "fdAT"
// MNG isn't supported by Firefox and IE because of lack of
market demand. Big surprise
// given nobody supports it! It is supported by Konqueror
though. Complex standard,
// not worth it for the time being.
Modified: branches/db4o/freenet/src/freenet/crypt/Yarrow.java
===================================================================
--- branches/db4o/freenet/src/freenet/crypt/Yarrow.java 2008-11-29 19:57:22 UTC
(rev 23985)
+++ branches/db4o/freenet/src/freenet/crypt/Yarrow.java 2008-11-29 20:04:11 UTC
(rev 23986)
@@ -99,7 +99,7 @@
else
seedfile = null;
if(reseedOnStartup) {
- entropy_init(seed);
+ entropy_init(seed, reseedOnStartup);
seedFromExternalStuff(canBlock);
/**
* If we don't reseed at this point, we will be
predictable,
@@ -184,26 +184,27 @@
consumeString(Long.toHexString(Runtime.getRuntime().totalMemory()));
}
- private void entropy_init(File seed) {
- Properties sys = System.getProperties();
- EntropySource startupEntropy = new EntropySource();
+ private void entropy_init(File seed, boolean reseedOnStartup) {
+ if(reseedOnStartup) {
+ Properties sys = System.getProperties();
+ EntropySource startupEntropy = new EntropySource();
- // Consume the system properties list
- for(Enumeration<?> enu = sys.propertyNames();
enu.hasMoreElements();) {
- String key = (String) enu.nextElement();
- consumeString(key);
- consumeString(sys.getProperty(key));
- }
+ // Consume the system properties list
+ for(Enumeration<?> enu = sys.propertyNames();
enu.hasMoreElements();) {
+ String key = (String) enu.nextElement();
+ consumeString(key);
+ consumeString(sys.getProperty(key));
+ }
- // Consume the local IP address
- try {
- consumeString(InetAddress.getLocalHost().toString());
- } catch(Exception e) {
- // Ignore
+ // Consume the local IP address
+ try {
+
consumeString(InetAddress.getLocalHost().toString());
+ } catch(Exception e) {
+ // Ignore
+ }
+ readStartupEntropy(startupEntropy);
}
- readStartupEntropy(startupEntropy);
-
read_seed(seed);
}
Modified: branches/db4o/freenet/src/freenet/node/Version.java
===================================================================
--- branches/db4o/freenet/src/freenet/node/Version.java 2008-11-29 19:57:22 UTC
(rev 23985)
+++ branches/db4o/freenet/src/freenet/node/Version.java 2008-11-29 20:04:11 UTC
(rev 23986)
@@ -24,7 +24,7 @@
public static final String protocolVersion = "1.0";
/** The build number of the current revision */
- private static final int buildNumber = 1181;
+ private static final int buildNumber = 1182;
/** Oldest build of Fred we will talk to */
private static final int oldLastGoodBuild = 1180;
Property changes on:
branches/db4o/freenet/src/freenet/support/BinaryBloomFilter.java
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/freenet/src/freenet/support/BinaryBloomFilter.java:22002-23794
+ /trunk/freenet/src/freenet/support/BinaryBloomFilter.java:22002-23821
Property changes on: branches/db4o/freenet/src/freenet/support/BloomFilter.java
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/freenet/src/freenet/support/BloomFilter.java:22002-23794
+ /trunk/freenet/src/freenet/support/BloomFilter.java:22002-23821
Property changes on:
branches/db4o/freenet/src/freenet/support/CountingBloomFilter.java
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/freenet/src/freenet/support/CountingBloomFilter.java:22002-23794
+ /trunk/freenet/src/freenet/support/CountingBloomFilter.java:22002-23821
Modified:
branches/db4o/freenet/src/freenet/support/compress/Bzip2Compressor.java
===================================================================
--- branches/db4o/freenet/src/freenet/support/compress/Bzip2Compressor.java
2008-11-29 19:57:22 UTC (rev 23985)
+++ branches/db4o/freenet/src/freenet/support/compress/Bzip2Compressor.java
2008-11-29 20:04:11 UTC (rev 23986)
@@ -3,20 +3,21 @@
* http://www.gnu.org/ for further details of the GPL. */
package freenet.support.compress;
+import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import org.apache.tools.bzip2.CBZip2InputStream;
+import org.apache.tools.bzip2.CBZip2OutputStream;
+
import freenet.support.Logger;
import freenet.support.api.Bucket;
import freenet.support.api.BucketFactory;
import freenet.support.io.CountedOutputStream;
-import org.apache.tools.bzip2.CBZip2InputStream;
-import org.apache.tools.bzip2.CBZip2OutputStream;
-
public class Bzip2Compressor implements Compressor {
public Bucket compress(Bucket data, BucketFactory bf, long
maxReadLength, long maxWriteLength) throws IOException,
CompressionOutputSizeException {
@@ -30,7 +31,7 @@
is = data.getInputStream();
os = output.getOutputStream();
CountedOutputStream cos = new CountedOutputStream(os);
- bz2os = new CBZip2OutputStream(cos);
+ bz2os = new CBZip2OutputStream(new
BufferedOutputStream(cos));
long read = 0;
// Bigger input buffer, so can compress all at once.
// Won't hurt on I/O either, although most OSs will
only return a page at a time.
Modified: branches/db4o/freenet/src/freenet/support/compress/GzipCompressor.java
===================================================================
--- branches/db4o/freenet/src/freenet/support/compress/GzipCompressor.java
2008-11-29 19:57:22 UTC (rev 23985)
+++ branches/db4o/freenet/src/freenet/support/compress/GzipCompressor.java
2008-11-29 20:04:11 UTC (rev 23986)
@@ -1,5 +1,6 @@
package freenet.support.compress;
+import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -25,7 +26,7 @@
GZIPOutputStream gos = null;
try {
is = data.getInputStream();
- os = output.getOutputStream();
+ os = new BufferedOutputStream(output.getOutputStream());
CountedOutputStream cos = new CountedOutputStream(os);
gos = new GZIPOutputStream(cos);
long read = 0;
Modified: branches/db4o/freenet/src/freenet/support/compress/LZMACompressor.java
===================================================================
--- branches/db4o/freenet/src/freenet/support/compress/LZMACompressor.java
2008-11-29 19:57:22 UTC (rev 23985)
+++ branches/db4o/freenet/src/freenet/support/compress/LZMACompressor.java
2008-11-29 20:04:11 UTC (rev 23986)
@@ -3,6 +3,8 @@
* http://www.gnu.org/ for further details of the GPL. */
package freenet.support.compress;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -10,49 +12,32 @@
import java.io.OutputStream;
import SevenZip.Compression.LZMA.Decoder;
-
+import SevenZip.Compression.LZMA.Encoder;
import freenet.support.Logger;
import freenet.support.api.Bucket;
import freenet.support.api.BucketFactory;
+import freenet.support.io.CountedInputStream;
import freenet.support.io.CountedOutputStream;
-import net.contrapunctus.lzma.LzmaInputStream;
-import net.contrapunctus.lzma.LzmaOutputStream;
public class LZMACompressor implements Compressor {
+ // Copied from EncoderThread. See below re licensing.
public Bucket compress(Bucket data, BucketFactory bf, long
maxReadLength, long maxWriteLength) throws IOException,
CompressionOutputSizeException {
- if(maxReadLength <= 0)
- throw new IllegalArgumentException();
- Bucket output = bf.makeBucket(maxWriteLength);
- InputStream is = null;
- OutputStream os = null;
- LzmaOutputStream lzmaOS = null;
- try {
- is = data.getInputStream();
- os = output.getOutputStream();
- CountedOutputStream cos = new CountedOutputStream(os);
- lzmaOS = new LzmaOutputStream(cos);
- long read = 0;
- // Bigger input buffer, so can compress all at once.
- // Won't hurt on I/O either, although most OSs will
only return a page at a time.
- byte[] buffer = new byte[32768];
- while(true) {
- int l = (int) Math.min(buffer.length,
maxReadLength - read);
- int x = l == 0 ? -1 : is.read(buffer, 0,
buffer.length);
- if(x <= -1) break;
- if(x == 0) throw new IOException("Returned zero
from read()");
- lzmaOS.write(buffer, 0, x);
- read += x;
- if(cos.written() > maxWriteLength)
- throw new
CompressionOutputSizeException();
- }
- lzmaOS.flush();
- os = null;
- } finally {
- if(is != null) is.close();
- if(lzmaOS != null) lzmaOS.close();
- else if(os != null) os.close();
- }
+ Bucket output;
+ output = bf.makeBucket(maxWriteLength);
+ if(Logger.shouldLog(Logger.MINOR, this))
+ Logger.minor(this, "Compressing "+data+" size
"+data.size()+" to new bucket "+output);
+ CountedInputStream is = new CountedInputStream(new
BufferedInputStream(data.getInputStream()));
+ CountedOutputStream os = new CountedOutputStream(new
BufferedOutputStream(output.getOutputStream()));
+ Encoder encoder = new Encoder();
+ encoder.SetEndMarkerMode( true );
+ encoder.SetDictionarySize( 1 << 20 );
+ // enc.WriteCoderProperties( out );
+ // 5d 00 00 10 00
+ encoder.Code( is, os, maxReadLength, maxWriteLength, null );
+ os.close();
+ if(Logger.shouldLog(Logger.MINOR, this))
+ Logger.minor(this, "Output: "+output+" size
"+output.size()+" read "+is.count()+" written "+os.written());
return output;
}
@@ -64,12 +49,12 @@
output = bf.makeBucket(maxLength);
if(Logger.shouldLog(Logger.MINOR, this))
Logger.minor(this, "Decompressing "+data+" size
"+data.size()+" to new bucket "+output);
- InputStream is = data.getInputStream();
- OutputStream os = output.getOutputStream();
+ CountedInputStream is = new CountedInputStream(new
BufferedInputStream(data.getInputStream()));
+ CountedOutputStream os = new CountedOutputStream(new
BufferedOutputStream(output.getOutputStream()));
decompress(is, os, maxLength, maxCheckSizeLength);
os.close();
if(Logger.shouldLog(Logger.MINOR, this))
- Logger.minor(this, "Output: "+output+" size
"+output.size());
+ Logger.minor(this, "Output: "+output+" size
"+output.size()+" read "+is.count()+" written "+os.written());
return output;
}
Modified: branches/db4o/freenet/src/freenet/support/io/CountedInputStream.java
===================================================================
--- branches/db4o/freenet/src/freenet/support/io/CountedInputStream.java
2008-11-29 19:57:22 UTC (rev 23985)
+++ branches/db4o/freenet/src/freenet/support/io/CountedInputStream.java
2008-11-29 20:04:11 UTC (rev 23986)
@@ -25,7 +25,7 @@
@Override
public int read(byte[] buf, int off, int len) throws IOException {
- int ret = super.read(buf, off, len);
+ int ret = in.read(buf, off, len);
if (ret != -1)
count += ret;
return ret;
@@ -33,7 +33,7 @@
@Override
public int read(byte[] buf) throws IOException {
- int ret = super.read(buf);
+ int ret = in.read(buf);
if (ret != -1)
count += ret;
return ret;
Modified: branches/db4o/freenet/test/freenet/crypt/YarrowTest.java
===================================================================
--- branches/db4o/freenet/test/freenet/crypt/YarrowTest.java 2008-11-29
19:57:22 UTC (rev 23985)
+++ branches/db4o/freenet/test/freenet/crypt/YarrowTest.java 2008-11-29
20:04:11 UTC (rev 23986)
@@ -24,27 +24,23 @@
(byte)0x1F, (byte)0xBD, (byte)0x91, (byte)0xEA, (byte)0xA2,
(byte)0xCD, (byte)0xD0, (byte)0xEB, (byte)0x37, (byte)0xF4
};
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ FileWriter fw = new FileWriter(SEED_FILE);
+ for(int i = 0; i < 256; i++)
+ fw.write(i);
+ fw.flush();
+ fw.close();
+ }
- public void testIt() {
- assertTrue(true);
+ @Override
+ protected void tearDown() throws Exception {
+ super.tearDown();
+ assertTrue(SEED_FILE.delete());
}
-
-// @Override
-// protected void setUp() throws Exception {
-// super.setUp();
-// FileWriter fw = new FileWriter(SEED_FILE);
-// for(int i = 0; i < 256; i++)
-// fw.write(i);
-// fw.flush();
-// fw.close();
-// }
-//
-// @Override
-// protected void tearDown() throws Exception {
-// super.tearDown();
-// assertTrue(SEED_FILE.delete());
-// }
-//
+
// public void testConsistencySeedFromFile() throws
NoSuchAlgorithmException, UnsupportedEncodingException {
// Yarrow y = new Yarrow(SEED_FILE, "SHA1", "Rijndael", false,
false, false);
// MessageDigest md = MessageDigest.getInstance("SHA-1");
@@ -57,21 +53,21 @@
// bytes = md.digest();
// assertEquals(new String(bytes, "UTF-8"), new
String(SEED_OUTPUT_YARROW_FILE, "UTF-8"));
// }
-//
-// public void testDouble() {
+
+ public void testDouble() {
+ Yarrow y = new Yarrow(SEED_FILE, "SHA1", "Rijndael", false,
false, false);
+ ScalarSampleStatistics sample = new ScalarSampleStatistics();
+
+ for(int i = 0; i < 10000; ++i) {
+ sample.add(y.nextDouble());
+ }
+
+ assertEquals(0.5, sample.getMean(), 0.02);
+ assertEquals(1.0 / (2.0 * Math.sqrt(3.0)),
sample.getStandardDeviation(), 0.002);
+ }
+
+ public void testNextInt() {
// Yarrow y = new Yarrow(SEED_FILE, "SHA1", "Rijndael", false,
false, false);
-// ScalarSampleStatistics sample = new ScalarSampleStatistics();
-//
-// for(int i = 0; i < 10000; ++i) {
-// sample.add(y.nextDouble());
-// }
-//
-// assertEquals(0.5, sample.getMean(), 0.02);
-// assertEquals(1.0 / (2.0 * Math.sqrt(3.0)),
sample.getStandardDeviation(), 0.002);
-// }
-//
-// public void testNextInt() {
-// Yarrow y = new Yarrow(SEED_FILE, "SHA1", "Rijndael", false,
false, false);
// for(int n = 1; n < 20; ++n) {
// int[] count = new int[n];
// for(int k = 0; k < 10000; ++k) {
@@ -85,5 +81,21 @@
// assertTrue(n * count[i] < 11100);
// }
// }
-// }
+ }
+
+ public void testNextBoolean() {
+ Yarrow y = new Yarrow(SEED_FILE, "SHA1", "Rijndael", false,
false, false);
+ int[] results = new int[2];
+ int RUNS = 1000000;
+ for(int i=0; i<RUNS; i++) {
+ if(y.nextBoolean())
+ results[0]++;
+ else
+ results[1]++;
+ }
+
+ assertEquals(RUNS, results[0]+results[1]);
+ assertTrue(results[0] > RUNS/2 - RUNS/1000);
+ assertTrue(results[1] > RUNS/2 - RUNS/1000);
+ }
}
Modified: branches/db4o/freenet/test/freenet/support/FieldsTest.java
===================================================================
--- branches/db4o/freenet/test/freenet/support/FieldsTest.java 2008-11-29
19:57:22 UTC (rev 23985)
+++ branches/db4o/freenet/test/freenet/support/FieldsTest.java 2008-11-29
20:04:11 UTC (rev 23986)
@@ -280,8 +280,10 @@
int outLong = Fields.bytesToInt(bytes, 0);
// We did dupplicate the code to avoid cross-dependancies
+ // TODO: Will need to be updated when next -ext is released
int outLongMT = MersenneTwister.bytesToInt(bytes, 0);
- assertEquals(outLong, outLongMT, 33685760);
+ assertEquals(outLong, outLongMT);
+ assertEquals(outLong, 33685760);
doTestRoundTripBytesArrayToInt(bytes);
Property changes on:
branches/db4o/freenet/test/freenet/support/io/MockInputStream.java
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk/freenet/test/freenet/support/io/MockInputStream.java:22002-23794
+ /trunk/freenet/test/freenet/support/io/MockInputStream.java:22002-23821
_______________________________________________
cvs mailing list
[email protected]
http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
