Author: toad
Date: 2005-10-24 17:08:08 +0000 (Mon, 24 Oct 2005)
New Revision: 7450
Added:
trunk/freenet/src/freenet/keys/KeyDecodeException.java
Modified:
trunk/freenet/src/freenet/client/ArchiveElement.java
trunk/freenet/src/freenet/client/ArchiveFailureException.java
trunk/freenet/src/freenet/client/ArchiveHandler.java
trunk/freenet/src/freenet/client/ArchiveManager.java
trunk/freenet/src/freenet/client/ArchiveRestartException.java
trunk/freenet/src/freenet/client/FetchException.java
trunk/freenet/src/freenet/client/Fetcher.java
trunk/freenet/src/freenet/client/FetcherContext.java
trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
trunk/freenet/src/freenet/client/Metadata.java
trunk/freenet/src/freenet/keys/CHKBlock.java
trunk/freenet/src/freenet/keys/CHKDecodeException.java
trunk/freenet/src/freenet/keys/KeyBlock.java
Log:
More work, mostly on archives.
Modified: trunk/freenet/src/freenet/client/ArchiveElement.java
===================================================================
--- trunk/freenet/src/freenet/client/ArchiveElement.java 2005-10-22
20:57:25 UTC (rev 7449)
+++ trunk/freenet/src/freenet/client/ArchiveElement.java 2005-10-24
17:08:08 UTC (rev 7450)
@@ -12,23 +12,26 @@
*/
public class ArchiveElement {
- ArchiveElement(ArchiveManager manager, FreenetURI uri, ClientKey ckey,
String filename) {
+ ArchiveElement(ArchiveManager manager, FreenetURI uri, ClientKey ckey,
String filename, short archiveType) {
this.manager = manager;
this.key = uri;
this.ckey = ckey;
this.filename = filename;
+ this.archiveType = archiveType;
}
final ArchiveManager manager;
final FreenetURI key;
final ClientKey ckey;
final String filename;
+ final short archiveType;
/**
* Fetch the element.
+ * If fetchContext is null, return null unless the data is cached.
* @throws ArchiveFailureException
*/
- public Bucket get(ArchiveContext archiveContext, FetcherContext
fetchContext) throws ArchiveFailureException {
+ public Bucket get(ArchiveContext archiveContext, FetcherContext
fetchContext, boolean inSplitZipManifest) throws ArchiveFailureException {
archiveContext.doLoopDetection(ckey);
// AFTER the loop check (possible deadlocks)
@@ -36,9 +39,13 @@
// Synchronized during I/O to avoid doing it twice
Bucket cached = manager.getCached(key, filename);
if(cached != null) return cached;
+ if(fetchContext == null) return null;
Fetcher fetcher = new Fetcher(key, fetchContext,
archiveContext);
- fetcher.realRun();
- manager.extractToCache(key, archiveContext);
+ FetchResult result =
fetcher.realRun(inSplitZipManifest);
+ if(result.succeeded())
+ manager.extractToCache(key, archiveType,
result.data, archiveContext);
+ else
+ throw new ArchiveFailureException("Fetch
failed");
}
}
}
Modified: trunk/freenet/src/freenet/client/ArchiveFailureException.java
===================================================================
--- trunk/freenet/src/freenet/client/ArchiveFailureException.java
2005-10-22 20:57:25 UTC (rev 7449)
+++ trunk/freenet/src/freenet/client/ArchiveFailureException.java
2005-10-24 17:08:08 UTC (rev 7450)
@@ -1,14 +1,23 @@
package freenet.client;
+import java.io.IOException;
+
/**
* Thrown when an archive operation fails.
*/
public class ArchiveFailureException extends Exception {
+ private static final long serialVersionUID = -5915105120222575469L;
+
public static final String TOO_MANY_LEVELS = "Too many archive levels";
public ArchiveFailureException(String message) {
super(message);
}
+ public ArchiveFailureException(String message, IOException e) {
+ super(message);
+ initCause(e);
+ }
+
}
Modified: trunk/freenet/src/freenet/client/ArchiveHandler.java
===================================================================
--- trunk/freenet/src/freenet/client/ArchiveHandler.java 2005-10-22
20:57:25 UTC (rev 7449)
+++ trunk/freenet/src/freenet/client/ArchiveHandler.java 2005-10-24
17:08:08 UTC (rev 7450)
@@ -10,10 +10,12 @@
private ArchiveManager manager;
private ClientKey key;
+ private short archiveType;
public ArchiveHandler(ArchiveManager manager, ClientKey key, short
archiveType) {
this.manager = manager;
this.key = key;
+ this.archiveType = archiveType;
}
public void finalize() {
@@ -24,17 +26,20 @@
* Get the metadata for this ZIP manifest, as a Bucket.
*/
public Bucket getMetadata(ArchiveContext archiveContext, FetcherContext
fetchContext) throws ArchiveFailureException, ArchiveRestartException {
- return get(".metadata", archiveContext, fetchContext);
+ return get(".metadata", archiveContext, fetchContext, false);
}
/**
* Get a file from this ZIP manifest, as a Bucket.
* If possible, read it from cache. If necessary, refetch the
* container and extract it. If that fails, throw.
+ * @param inSplitZipManifest If true, indicates that the key points to
a splitfile zip manifest,
+ * which means that we need to pass a flag to the fetcher to tell it to
pretend it was a straight
+ * splitfile.
*/
- public synchronized Bucket get(String internalName, ArchiveContext
archiveContext, FetcherContext fetchContext) throws ArchiveFailureException,
ArchiveRestartException {
+ public synchronized Bucket get(String internalName, ArchiveContext
archiveContext, FetcherContext fetchContext, boolean inSplitZipManifest) throws
ArchiveFailureException, ArchiveRestartException {
ArchiveElement element =
- manager.makeElement(key, internalName);
- return element.get(archiveContext, fetchContext);
+ manager.makeElement(key, internalName, archiveType);
+ return element.get(archiveContext, fetchContext,
inSplitZipManifest);
}
}
Modified: trunk/freenet/src/freenet/client/ArchiveManager.java
===================================================================
--- trunk/freenet/src/freenet/client/ArchiveManager.java 2005-10-22
20:57:25 UTC (rev 7449)
+++ trunk/freenet/src/freenet/client/ArchiveManager.java 2005-10-24
17:08:08 UTC (rev 7450)
@@ -1,13 +1,19 @@
package freenet.client;
import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.util.LinkedHashMap;
import java.util.Set;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
import freenet.keys.ClientKey;
import freenet.keys.FreenetURI;
import freenet.support.Bucket;
import freenet.support.LRUHashtable;
+import freenet.support.Logger;
/**
* Cache of recently decoded archives:
@@ -18,14 +24,19 @@
*/
public class ArchiveManager {
- ArchiveManager(int maxHandlers, long maxCachedData, File cacheDir) {
+ ArchiveManager(int maxHandlers, long maxCachedData, long
maxArchiveSize, long maxArchivedFileSize, File cacheDir) {
maxArchiveHandlers = maxHandlers;
archiveHandlers = new LRUHashtable();
this.maxCachedData = maxCachedData;
this.cacheDir = cacheDir;
storedData = new LRUHashtable();
+ this.maxArchiveSize = maxArchiveSize;
+ this.maxArchivedFileSize = maxArchivedFileSize;
}
+ final long maxArchiveSize;
+ final long maxArchivedFileSize;
+
// ArchiveHandler's
final int maxArchiveHandlers;
@@ -97,7 +108,7 @@
MyKey key;
boolean finalized;
// FIXME implement
-
+
public Bucket dataAsBucket() {
// FIXME implement
}
@@ -108,4 +119,60 @@
}
}
+ /**
+ * Extract data to cache.
+ * @param key The key the data was fetched from.
+ * @param archiveType The archive type. Must be Metadata.ARCHIVE_ZIP.
+ * @param data The actual data fetched.
+ * @param archiveContext The context for the whole fetch process.
+ * @throws ArchiveFailureException
+ */
+ public void extractToCache(FreenetURI key, short archiveType, Bucket
data, ArchiveContext archiveContext) throws ArchiveFailureException {
+ if(data.size() > maxArchiveSize)
+ throw new ArchiveFailureException("Archive too big");
+ if(archiveType != Metadata.ARCHIVE_ZIP)
+ throw new ArchiveFailureException("Unknown or
unsupported archive algorithm "+archiveType);
+ InputStream is = null;
+ try {
+ is = data.getInputStream();
+ ZipInputStream zis = new ZipInputStream(is);
+ ZipEntry entry = zis.getNextEntry();
+ byte[] buf = new byte[4096];
+outer: while(entry != null) {
+ entry = zis.getNextEntry();
+ String name = entry.getName();
+ long size = entry.getSize();
+ if(size > maxArchivedFileSize) {
+ addErrorElement(key, name);
+ } else {
+ // Read the element
+ long realLen = 0;
+ Bucket output =
makeTempStoreBucket(size);
+ OutputStream out =
output.getOutputStream();
+ int readBytes;
+inner: while((readBytes = zis.read(buf)) > 0) {
+ out.write(buf, 0, readBytes);
+ readBytes += realLen;
+ if(readBytes >
maxArchivedFileSize) {
+ addErrorElement(key,
name);
+ out.close();
+
dumpTempStoreBucket(output);
+ continue outer;
+ }
+ }
+ out.close();
+ addStoreElement(key, name, output);
+ }
+ }
+ } catch (IOException e) {
+ throw new ArchiveFailureException("Error reading
archive: "+e.getMessage(), e);
+ } finally {
+ if(is != null)
+ try {
+ is.close();
+ } catch (IOException e) {
+ Logger.error(this, "Failed to close
stream: "+e, e);
+ }
+ }
+ }
}
Modified: trunk/freenet/src/freenet/client/ArchiveRestartException.java
===================================================================
--- trunk/freenet/src/freenet/client/ArchiveRestartException.java
2005-10-22 20:57:25 UTC (rev 7449)
+++ trunk/freenet/src/freenet/client/ArchiveRestartException.java
2005-10-24 17:08:08 UTC (rev 7450)
@@ -7,4 +7,8 @@
*/
public class ArchiveRestartException extends Exception {
+ public ArchiveRestartException(String msg) {
+ super(msg);
+ }
+
}
Modified: trunk/freenet/src/freenet/client/FetchException.java
===================================================================
--- trunk/freenet/src/freenet/client/FetchException.java 2005-10-22
20:57:25 UTC (rev 7449)
+++ trunk/freenet/src/freenet/client/FetchException.java 2005-10-24
17:08:08 UTC (rev 7450)
@@ -30,5 +30,10 @@
static final int INVALID_METADATA = 4;
/** Got an ArchiveFailureException */
static final int ARCHIVE_FAILURE = 5;
-
+ /** Failed to decode a block */
+ static final int BLOCK_DECODE_ERROR = 6;
+ /** Too many split metadata levels */
+ static final int TOO_MANY_METADATA_LEVELS = 7;
+ /** Too many archive restarts */
+ static final int TOO_MANY_ARCHIVE_RESTARTS = 8;
}
Modified: trunk/freenet/src/freenet/client/Fetcher.java
===================================================================
--- trunk/freenet/src/freenet/client/Fetcher.java 2005-10-22 20:57:25 UTC
(rev 7449)
+++ trunk/freenet/src/freenet/client/Fetcher.java 2005-10-24 17:08:08 UTC
(rev 7450)
@@ -5,6 +5,7 @@
import freenet.keys.ClientKey;
import freenet.keys.FreenetURI;
import freenet.keys.KeyBlock;
+import freenet.keys.KeyDecodeException;
import freenet.support.Bucket;
import freenet.support.Logger;
@@ -15,7 +16,7 @@
final FreenetURI origURI;
final FetcherContext ctx;
- final ArchiveContext archiveContext;
+ ArchiveContext archiveContext;
public Fetcher(FreenetURI uri, FetcherContext ctx, ArchiveContext
archiveContext) {
this.origURI = uri;
@@ -24,10 +25,11 @@
}
public FetchResult run() throws FetchException {
- while(true) {
+ for(int i=0;i<ctx.maxArchiveRestarts;i++) {
try {
- return realRun();
+ return realRun(false);
} catch (ArchiveRestartException e) {
+ archiveContext = new ArchiveContext();
continue;
} catch (MetadataParseException e) {
throw new FetchException(e);
@@ -37,16 +39,18 @@
throw new FetchException(e);
}
}
+ throw new
FetchException(FetchException.TOO_MANY_ARCHIVE_RESTARTS);
}
/**
* Run the actual fetch.
+ * @param overrideZipManifest
* @return The result of the fetch - successful or not.
* @throws FetchException
* @throws MetadataParseException
* @throws ArchiveFailureException
*/
- public FetchResult realRun() throws FetchException,
ArchiveRestartException, MetadataParseException, ArchiveFailureException {
+ public FetchResult realRun(boolean overrideZipManifest) throws
FetchException, ArchiveRestartException, MetadataParseException,
ArchiveFailureException {
FreenetURI uri = origURI;
ClientKey key = ClientKey.get(origURI);
ClientMetadata dm = new ClientMetadata();
@@ -59,7 +63,12 @@
// Fetch the first key
KeyBlock block = ctx.client.getKey(key);
- byte[] data = block.decode(key);
+ byte[] data;
+ try {
+ data = block.decode(key);
+ } catch (KeyDecodeException e1) {
+ throw new
FetchException(FetchException.BLOCK_DECODE_ERROR);
+ }
if(!key.isMetadata()) {
// Just return the data
@@ -74,8 +83,17 @@
// This will throw if it finds an error, including
semi-errors
// such as too-big-indirect-metadata
Metadata metadata = Metadata.construct(data);
+ if(overrideZipManifest) {
+ if(metadata.isArchiveManifest())
+ metadata.setSimpleRedirect();
+ else
+ throw new
ArchiveRestartException("Override zip manifest set but not a zip manifest!");
+ }
while(true) {
+
+ /** Is set false every time we get a new key. */
+ boolean inSplitZipManifest = false;
if(metadata == null && key != null) {
// Try next key
@@ -114,11 +132,23 @@
*
* Now, retreive the data
*/
- Bucket result =
zip.get(metadata.getZIPInternalName(), archiveContext, ctx);
+ Bucket result =
zip.get(metadata.getZIPInternalName(), archiveContext, ctx, inSplitZipManifest);
dm.mergeNoOverwrite(metadata.getClientMetadata());
return new FetchResult(dm, result);
} else if(metadata.isSplitfile()) {
+ if(metadata.isArchiveManifest()) {
+ // Check the cache first
+ zip =
ctx.archiveManager.makeHandler(key, metadata.getArchiveType());
+ // Future
ArchiveInternalRedirects will point to *self*
+ inSplitZipManifest = true;
+ Bucket metadataBucket =
zip.getMetadata(archiveContext, null);
+ if(metadataBucket != null) {
+ metadata =
Metadata.construct(metadataBucket);
+ continue;
+ }
+ }
+
int j;
for(j=0;j<ctx.maxLevels;j++) {
SplitFetcher sf = new
SplitFetcher(metadata, ctx.maxTempLength, archiveContext, ctx);
@@ -131,7 +161,8 @@
if(!metadata.isMultiLevelMetadata())
break; // try
the new metadata
} else
if(metadata.isArchiveManifest()) {
- zip =
ctx.archiveManager.makeHandler(key, metadata.getArchiveType());
+ // Use the new metadata
+ // ZIP points to
current key
Bucket metadataBucket =
zip.getMetadata(archiveContext, ctx);
metadata =
Metadata.construct(metadataBucket);
break;
@@ -140,6 +171,7 @@
}
} // loop (splitfile levels)
if(j>=ctx.maxLevels) {
+ throw new
FetchException(FetchException.TOO_MANY_METADATA_LEVELS);
// Too many levels
// FIXME: throw something
}
Modified: trunk/freenet/src/freenet/client/FetcherContext.java
===================================================================
--- trunk/freenet/src/freenet/client/FetcherContext.java 2005-10-22
20:57:25 UTC (rev 7449)
+++ trunk/freenet/src/freenet/client/FetcherContext.java 2005-10-24
17:08:08 UTC (rev 7450)
@@ -14,16 +14,19 @@
final int maxArchiveRecursionLevel;
final ArchiveManager archiveManager;
final BucketFactory bucketFactory;
+ final int maxArchiveRestarts;
public FetcherContext(SimpleLowLevelClient client, long curMaxLength,
- long curMaxTempLength, int maxRedirects, int maxLevels,
int maxArchives,
- ArchiveManager archiveManager, BucketFactory
bucketFactory) {
+ long curMaxTempLength, int maxRedirects, int maxLevels,
int maxArchives,
+ int maxArchiveRestarts, ArchiveManager archiveManager,
+ BucketFactory bucketFactory) {
this.client = client;
this.maxOutputLength = curMaxLength;
this.maxTempLength = curMaxTempLength;
this.maxRedirects = maxRedirects;
this.maxLevels = maxLevels;
this.maxArchiveRecursionLevel = maxArchives;
+ this.maxArchiveRestarts = maxArchiveRestarts;
this.archiveManager = archiveManager;
this.bucketFactory = bucketFactory;
}
Modified: trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
===================================================================
--- trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
2005-10-22 20:57:25 UTC (rev 7449)
+++ trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
2005-10-24 17:08:08 UTC (rev 7450)
@@ -11,9 +11,10 @@
private final BucketFactory bucketFactory;
private long curMaxLength;
private long curMaxTempLength;
- public static final int MAX_REDIRECTS = 10;
- public static final int MAX_METADATA_LEVELS = 5;
- public static final int MAX_ARCHIVE_LEVELS = 5;
+ static final int MAX_REDIRECTS = 10;
+ static final int MAX_METADATA_LEVELS = 5;
+ static final int MAX_ARCHIVE_LEVELS = 5;
+ static final int MAX_ARCHIVE_RESTARTS = 2;
public HighLevelSimpleClientImpl(SimpleLowLevelClient client,
ArchiveManager mgr, BucketFactory bf) {
this.client = client;
@@ -31,7 +32,7 @@
public FetchResult fetch(FreenetURI uri) throws FetchException {
FetcherContext context = new FetcherContext(client,
curMaxLength, curMaxLength,
- MAX_REDIRECTS, MAX_METADATA_LEVELS,
MAX_ARCHIVE_LEVELS, archiveManager, bucketFactory);
+ MAX_REDIRECTS, MAX_METADATA_LEVELS,
MAX_ARCHIVE_LEVELS, MAX_ARCHIVE_RESTARTS, archiveManager, bucketFactory);
Fetcher f = new Fetcher(uri, context, new ArchiveContext());
return f.run();
}
Modified: trunk/freenet/src/freenet/client/Metadata.java
===================================================================
--- trunk/freenet/src/freenet/client/Metadata.java 2005-10-22 20:57:25 UTC
(rev 7449)
+++ trunk/freenet/src/freenet/client/Metadata.java 2005-10-24 17:08:08 UTC
(rev 7450)
@@ -405,4 +405,11 @@
public short getArchiveType() {
return archiveType;
}
+
+ /** Change the document type to a simple redirect. Used by the archive
code
+ * to fetch split ZIP manifests.
+ */
+ public void setSimpleRedirect() {
+ documentType = SIMPLE_REDIRECT;
+ }
}
Modified: trunk/freenet/src/freenet/keys/CHKBlock.java
===================================================================
--- trunk/freenet/src/freenet/keys/CHKBlock.java 2005-10-22 20:57:25 UTC
(rev 7449)
+++ trunk/freenet/src/freenet/keys/CHKBlock.java 2005-10-24 17:08:08 UTC
(rev 7450)
@@ -79,6 +79,12 @@
// Otherwise it checks out
}
+ public byte[] decode(ClientKey key) throws KeyDecodeException {
+ if(!(key instanceof ClientCHK))
+ throw new KeyDecodeException("Not a CHK!: "+key);
+ return decode((ClientCHK)key);
+ }
+
/**
* Decode the CHK and recover the original data
* @return the original data
Modified: trunk/freenet/src/freenet/keys/CHKDecodeException.java
===================================================================
--- trunk/freenet/src/freenet/keys/CHKDecodeException.java 2005-10-22
20:57:25 UTC (rev 7449)
+++ trunk/freenet/src/freenet/keys/CHKDecodeException.java 2005-10-24
17:08:08 UTC (rev 7450)
@@ -5,7 +5,7 @@
*
* Exception thrown when decode fails.
*/
-public class CHKDecodeException extends Exception {
+public class CHKDecodeException extends KeyDecodeException {
/**
*
Modified: trunk/freenet/src/freenet/keys/KeyBlock.java
===================================================================
--- trunk/freenet/src/freenet/keys/KeyBlock.java 2005-10-22 20:57:25 UTC
(rev 7449)
+++ trunk/freenet/src/freenet/keys/KeyBlock.java 2005-10-24 17:08:08 UTC
(rev 7450)
@@ -6,6 +6,6 @@
public interface KeyBlock {
/** Decode with the key */
- byte[] decode(ClientKey key);
+ byte[] decode(ClientKey key) throws KeyDecodeException;
}
Added: trunk/freenet/src/freenet/keys/KeyDecodeException.java
===================================================================
--- trunk/freenet/src/freenet/keys/KeyDecodeException.java 2005-10-22
20:57:25 UTC (rev 7449)
+++ trunk/freenet/src/freenet/keys/KeyDecodeException.java 2005-10-24
17:08:08 UTC (rev 7450)
@@ -0,0 +1,24 @@
+package freenet.keys;
+
+/**
+ * Base class for decode exceptions.
+ */
+public class KeyDecodeException extends Exception {
+
+ public KeyDecodeException(String message) {
+ super(message);
+ }
+
+ public KeyDecodeException() {
+ super();
+ }
+
+ public KeyDecodeException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public KeyDecodeException(Throwable cause) {
+ super(cause);
+ }
+
+}
_______________________________________________
cvs mailing list
[email protected]
http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs
