Author: toad
Date: 2007-06-16 13:19:29 +0000 (Sat, 16 Jun 2007)
New Revision: 13622
Added:
trunk/freenet/src/freenet/client/ArchiveExtractCallback.java
trunk/freenet/src/freenet/support/io/MultiReaderBucket.java
Modified:
trunk/freenet/src/freenet/client/ArchiveManager.java
trunk/freenet/src/freenet/client/ArchiveStoreContext.java
trunk/freenet/src/freenet/client/ArchiveStoreItem.java
trunk/freenet/src/freenet/client/ErrorArchiveStoreItem.java
trunk/freenet/src/freenet/client/RealArchiveStoreItem.java
trunk/freenet/src/freenet/client/async/SingleFileFetcher.java
Log:
Solve various problems in archive extraction:
- Prevent infinite recursion.
- Prevent race conditions involving data being repeatedly extracted and deleted.
- Deal correctly with the extracted data not fitting in the cache (thus the
cache can be tiny if necessary, and containers can be huge e.g. inserting a
kernel tarball as an implicit container).
- Avoid unnecessary copying.
Added: trunk/freenet/src/freenet/client/ArchiveExtractCallback.java
===================================================================
--- trunk/freenet/src/freenet/client/ArchiveExtractCallback.java
(rev 0)
+++ trunk/freenet/src/freenet/client/ArchiveExtractCallback.java
2007-06-16 13:19:29 UTC (rev 13622)
@@ -0,0 +1,15 @@
+package freenet.client;
+
+import freenet.support.api.Bucket;
+
+/** Called when we have extracted an archive, and a specified file either is
+ * or isn't in it. */
+public interface ArchiveExtractCallback {
+
+ /** Got the data */
+ public void gotBucket(Bucket data);
+
+ /** Not in the archive */
+ public void notInArchive();
+
+}
Modified: trunk/freenet/src/freenet/client/ArchiveManager.java
===================================================================
--- trunk/freenet/src/freenet/client/ArchiveManager.java 2007-06-15
23:36:51 UTC (rev 13621)
+++ trunk/freenet/src/freenet/client/ArchiveManager.java 2007-06-16
13:19:29 UTC (rev 13622)
@@ -34,6 +34,7 @@
*/
public class ArchiveManager {
+ public static final String METADATA_NAME = ".metadata";
private static boolean logMINOR;
/**
@@ -145,7 +146,7 @@
storedData.push(k, asi);
}
if(logMINOR) Logger.minor(this, "Found data");
- return asi.getDataOrThrow();
+ return asi.getReaderBucket();
}
/**
@@ -165,15 +166,18 @@
* @param data The actual data fetched.
* @param archiveContext The context for the whole fetch process.
* @param ctx The ArchiveStoreContext for this key.
+ * @param element A particular element that the caller is especially
interested in, or null.
+ * @param callback A callback to be called if we find that element, or
if we don't.
* @throws ArchiveFailureException If we could not extract the data, or
it was too big, etc.
* @throws ArchiveRestartException
* @throws ArchiveRestartException If the request needs to be restarted
because the archive
* changed.
*/
- public void extractToCache(FreenetURI key, short archiveType, Bucket
data, ArchiveContext archiveContext, ArchiveStoreContext ctx) throws
ArchiveFailureException, ArchiveRestartException {
+ public void extractToCache(FreenetURI key, short archiveType, Bucket
data, ArchiveContext archiveContext, ArchiveStoreContext ctx, String element,
ArchiveExtractCallback callback) throws ArchiveFailureException,
ArchiveRestartException {
logMINOR = Logger.shouldLog(Logger.MINOR, this);
+ boolean gotElement = false;
if(logMINOR) Logger.minor(this, "Extracting "+key);
ctx.onExtract();
ctx.removeAllCachedItems(); // flush cache anyway
@@ -249,16 +253,30 @@
out.close();
if(name.equals(".metadata"))
gotMetadata = true;
- addStoreElement(ctx, key, name, temp);
+ ArchiveStoreItem item =
addStoreElement(ctx, key, name, temp);
+ if((!gotElement) && element != null &&
name.equals(element)) {
+ gotElement = true;
+ // Let it throw, if it does
something is drastically wrong
+
callback.gotBucket(item.getReaderBucket());
+ }
names.add(name);
}
}
// If no metadata, generate some
if(!gotMetadata) {
- generateMetadata(ctx, key, names);
+ ArchiveStoreItem item = generateMetadata(ctx,
key, names);
+ if(element != null && (!gotElement) &&
element.equals(METADATA_NAME)) {
+ gotElement = true;
+ // Let it throw, if it does something
is drastically wrong
+
callback.gotBucket(item.getReaderBucket());
+ }
}
if(throwAtExit) throw new
ArchiveRestartException("Archive changed on re-fetch");
+
+ if((!gotElement) && element != null)
+ callback.notInArchive();
+
} catch (IOException e) {
throw new ArchiveFailureException("Error reading
archive: "+e.getMessage(), e);
} finally {
@@ -279,7 +297,7 @@
* @param names Set of names in the archive.
* @throws ArchiveFailureException
*/
- private void generateMetadata(ArchiveStoreContext ctx, FreenetURI key,
HashSet names) throws ArchiveFailureException {
+ private ArchiveStoreItem generateMetadata(ArchiveStoreContext ctx,
FreenetURI key, HashSet names) throws ArchiveFailureException {
/* What we have to do is to:
* - Construct a filesystem tree of the names.
* - Turn each level of the tree into a Metadata object,
including those below it, with
@@ -304,8 +322,7 @@
OutputStream os =
element.bucket.getOutputStream();
os.write(buf);
os.close();
- addStoreElement(ctx, key, ".metadata", element);
- break;
+ return addStoreElement(ctx, key, ".metadata",
element);
} catch (MetadataUnresolvedException e) {
try {
x = resolve(e, x, element, ctx, key);
@@ -383,13 +400,14 @@
/**
* Add a store element.
*/
- private void addStoreElement(ArchiveStoreContext ctx, FreenetURI key,
String name, TempStoreElement temp) {
+ private ArchiveStoreItem addStoreElement(ArchiveStoreContext ctx,
FreenetURI key, String name, TempStoreElement temp) {
RealArchiveStoreItem element = new RealArchiveStoreItem(this,
ctx, key, name, temp);
if(logMINOR) Logger.minor(this, "Adding store element:
"+element+" ( "+key+ ' ' +name+" size "+element.spaceUsed()+" )");
synchronized (storedData) {
storedData.push(element.key, element);
}
trimStoredData();
+ return element;
}
/**
Modified: trunk/freenet/src/freenet/client/ArchiveStoreContext.java
===================================================================
--- trunk/freenet/src/freenet/client/ArchiveStoreContext.java 2007-06-15
23:36:51 UTC (rev 13621)
+++ trunk/freenet/src/freenet/client/ArchiveStoreContext.java 2007-06-16
13:19:29 UTC (rev 13622)
@@ -46,6 +46,8 @@
/**
* Fetch a file in an archive.
+ * @return A Bucket containing the data. This will not be freed until
the
+ * client is finished with it i.e. calls free() or it is finalized.
*/
public Bucket get(String internalName, ArchiveContext archiveContext,
ClientMetadata dm, int recursionLevel,
boolean dontEnterImplicitArchives) throws
ArchiveFailureException, ArchiveRestartException, MetadataParseException,
FetchException {
@@ -139,8 +141,8 @@
return key;
}
- public void extractToCache(Bucket bucket, ArchiveContext actx) throws
ArchiveFailureException, ArchiveRestartException {
- manager.extractToCache(key, archiveType, bucket, actx, this);
+ public void extractToCache(Bucket bucket, ArchiveContext actx, String
element, ArchiveExtractCallback callback) throws ArchiveFailureException,
ArchiveRestartException {
+ manager.extractToCache(key, archiveType, bucket, actx, this,
element, callback);
}
public void onExtract() {
Modified: trunk/freenet/src/freenet/client/ArchiveStoreItem.java
===================================================================
--- trunk/freenet/src/freenet/client/ArchiveStoreItem.java 2007-06-15
23:36:51 UTC (rev 13621)
+++ trunk/freenet/src/freenet/client/ArchiveStoreItem.java 2007-06-16
13:19:29 UTC (rev 13622)
@@ -39,4 +39,10 @@
* Return the amount of cache space used by the item.
*/
abstract long spaceUsed();
+
+ /**
+ * Get the data as a Bucket, and guarantee that it won't be freed until
the
+ * returned object is either finalized or freed.
+ */
+ abstract Bucket getReaderBucket() throws ArchiveFailureException;
}
Modified: trunk/freenet/src/freenet/client/ErrorArchiveStoreItem.java
===================================================================
--- trunk/freenet/src/freenet/client/ErrorArchiveStoreItem.java 2007-06-15
23:36:51 UTC (rev 13621)
+++ trunk/freenet/src/freenet/client/ErrorArchiveStoreItem.java 2007-06-16
13:19:29 UTC (rev 13622)
@@ -35,5 +35,9 @@
public long spaceUsed() {
return 0;
}
+
+ Bucket getReaderBucket() throws ArchiveFailureException {
+ throw new ArchiveFailureException(error);
+ }
}
Modified: trunk/freenet/src/freenet/client/RealArchiveStoreItem.java
===================================================================
--- trunk/freenet/src/freenet/client/RealArchiveStoreItem.java 2007-06-15
23:36:51 UTC (rev 13621)
+++ trunk/freenet/src/freenet/client/RealArchiveStoreItem.java 2007-06-16
13:19:29 UTC (rev 13622)
@@ -7,16 +7,15 @@
import freenet.keys.FreenetURI;
import freenet.support.api.Bucket;
-import freenet.support.io.FileBucket;
import freenet.support.io.FileUtil;
-import freenet.support.io.PaddedEphemerallyEncryptedBucket;
+import freenet.support.io.MultiReaderBucket;
class RealArchiveStoreItem extends ArchiveStoreItem {
private final ArchiveManager manager;
private final File myFilename;
- private final PaddedEphemerallyEncryptedBucket bucket;
- private final FileBucket underBucket;
+ private final MultiReaderBucket mb;
+ private final Bucket bucket;
private final long spaceUsed;
/**
@@ -29,11 +28,11 @@
RealArchiveStoreItem(ArchiveManager manager, ArchiveStoreContext ctx,
FreenetURI key2, String realName, TempStoreElement temp) {
super(new ArchiveKey(key2, realName), ctx);
this.manager = manager;
- this.bucket = temp.bucket;
- this.underBucket = temp.underBucket;
- underBucket.setReadOnly();
- this.myFilename = underBucket.getFile();
- spaceUsed = FileUtil.estimateUsage(myFilename,
underBucket.size());
+ mb = new MultiReaderBucket(temp.bucket);
+ this.bucket = mb.getReaderBucket();
+ temp.underBucket.setReadOnly();
+ this.myFilename = temp.underBucket.getFile();
+ spaceUsed = FileUtil.estimateUsage(myFilename,
temp.underBucket.size());
this.manager.incrementSpace(spaceUsed);
}
@@ -59,10 +58,14 @@
}
void innerClose() {
- underBucket.free();
+ bucket.free();
}
Bucket getDataOrThrow() throws ArchiveFailureException {
return dataAsBucket();
}
+
+ Bucket getReaderBucket() throws ArchiveFailureException {
+ return mb.getReaderBucket();
+ }
}
Modified: trunk/freenet/src/freenet/client/async/SingleFileFetcher.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SingleFileFetcher.java
2007-06-15 23:36:51 UTC (rev 13621)
+++ trunk/freenet/src/freenet/client/async/SingleFileFetcher.java
2007-06-16 13:19:29 UTC (rev 13622)
@@ -8,7 +8,9 @@
import java.util.LinkedList;
import freenet.client.ArchiveContext;
+import freenet.client.ArchiveExtractCallback;
import freenet.client.ArchiveFailureException;
+import freenet.client.ArchiveManager;
import freenet.client.ArchiveRestartException;
import freenet.client.ArchiveStoreContext;
import freenet.client.ClientMetadata;
@@ -274,7 +276,31 @@
throw new
FetchException(FetchException.BUCKET_ERROR, e);
}
} else {
- fetchArchive(false, archiveMetadata);
// will result in this function being called again
+ fetchArchive(false, archiveMetadata,
ArchiveManager.METADATA_NAME, new ArchiveExtractCallback() {
+ public void gotBucket(Bucket
data) {
+ try {
+ metadata =
Metadata.construct(data);
+ } catch (IOException e)
{
+ // Bucket error?
+ onFailure(new
FetchException(FetchException.BUCKET_ERROR, e));
+ }
+ try {
+
handleMetadata();
+ } catch
(MetadataParseException e) {
+
SingleFileFetcher.this.onFailure(new FetchException(e));
+ } catch (FetchException
e) {
+
e.setNotFinalizedSize();
+
SingleFileFetcher.this.onFailure(e);
+ } catch
(ArchiveFailureException e) {
+
SingleFileFetcher.this.onFailure(new FetchException(e));
+ } catch
(ArchiveRestartException e) {
+
SingleFileFetcher.this.onFailure(new FetchException(e));
+ }
+ }
+ public void notInArchive() {
+ onFailure(new
FetchException(FetchException.INTERNAL_ERROR, "No metadata in container! Cannot
happen as ArchiveManager should synthesise some!"));
+ }
+ }); // will result in this function
being called again
return;
}
continue;
@@ -289,17 +315,15 @@
Bucket dataBucket = ah.get(filename, actx,
null, recursionLevel+1, true);
if(dataBucket != null) {
if(logMINOR) Logger.minor(this,
"Returning data");
- // The client may free it, which is
bad, or it may hang on to it for so long that it gets
- // freed by us, which is also bad.
- // So copy it.
- // FIXME this is stupid, reconsider how
we determine when to free buckets; refcounts maybe?
Bucket out;
try {
- if(returnBucket != null)
+ // Data will not be freed until
client is finished with it.
+ if(returnBucket != null) {
out = returnBucket;
- else
- out =
ctx.bucketFactory.makeBucket(dataBucket.size());
- BucketTools.copy(dataBucket,
out);
+
BucketTools.copy(dataBucket, out);
+ } else {
+ out = dataBucket;
+ }
} catch (IOException e) {
onFailure(new
FetchException(FetchException.BUCKET_ERROR));
return;
@@ -312,7 +336,29 @@
// Metadata cannot contain pointers to
files which don't exist.
// We enforce this in ArchiveHandler.
// Therefore, the archive needs to be
fetched.
- fetchArchive(true, archiveMetadata);
+ fetchArchive(true, archiveMetadata,
filename, new ArchiveExtractCallback() {
+ public void gotBucket(Bucket
data) {
+ if(logMINOR)
Logger.minor(this, "Returning data");
+ Bucket out;
+ try {
+ // Data will
not be freed until client is finished with it.
+ if(returnBucket
!= null) {
+ out =
returnBucket;
+
BucketTools.copy(data, out);
+ } else {
+ out =
data;
+ }
+ } catch (IOException e)
{
+ onFailure(new
FetchException(FetchException.BUCKET_ERROR));
+ return;
+ }
+ // Return the data
+ onSuccess(new
FetchResult(clientMetadata, out));
+ }
+ public void notInArchive() {
+ onFailure(new
FetchException(FetchException.NOT_IN_ARCHIVE));
+ }
+ });
// Will call back into this function
when it has been fetched.
return;
}
@@ -440,7 +486,7 @@
decompressors.addLast(codec);
}
- private void fetchArchive(boolean forData, Metadata meta) throws
FetchException, MetadataParseException, ArchiveFailureException,
ArchiveRestartException {
+ private void fetchArchive(boolean forData, Metadata meta, String
element, ArchiveExtractCallback callback) throws FetchException,
MetadataParseException, ArchiveFailureException, ArchiveRestartException {
if(logMINOR) Logger.minor(this, "fetchArchive()");
// Fetch the archive
// How?
@@ -451,7 +497,7 @@
Metadata newMeta = (Metadata) meta.clone();
newMeta.setSimpleRedirect();
SingleFileFetcher f;
- f = new SingleFileFetcher(this, newMeta, new
ArchiveFetcherCallback(forData), new FetchContext(ctx,
FetchContext.SET_RETURN_ARCHIVES, true));
+ f = new SingleFileFetcher(this, newMeta, new
ArchiveFetcherCallback(forData, element, callback), new FetchContext(ctx,
FetchContext.SET_RETURN_ARCHIVES, true));
f.handleMetadata();
// When it is done (if successful), the ArchiveCallback will
re-call this function.
// Which will then discover that the metadata *is* available.
@@ -461,14 +507,18 @@
class ArchiveFetcherCallback implements GetCompletionCallback {
private final boolean wasFetchingFinalData;
+ private final String element;
+ private final ArchiveExtractCallback callback;
- ArchiveFetcherCallback(boolean wasFetchingFinalData) {
+ ArchiveFetcherCallback(boolean wasFetchingFinalData, String
element, ArchiveExtractCallback cb) {
this.wasFetchingFinalData = wasFetchingFinalData;
+ this.element = element;
+ this.callback = cb;
}
public void onSuccess(FetchResult result, ClientGetState state)
{
try {
- ah.extractToCache(result.asBucket(), actx);
+ ah.extractToCache(result.asBucket(), actx,
element, callback);
} catch (ArchiveFailureException e) {
SingleFileFetcher.this.onFailure(new
FetchException(e));
return;
Added: trunk/freenet/src/freenet/support/io/MultiReaderBucket.java
===================================================================
--- trunk/freenet/src/freenet/support/io/MultiReaderBucket.java
(rev 0)
+++ trunk/freenet/src/freenet/support/io/MultiReaderBucket.java 2007-06-16
13:19:29 UTC (rev 13622)
@@ -0,0 +1,83 @@
+/* This code is part of Freenet. It is distributed under the GNU General
+ * Public License, version 2 (or at your option any later version). See
+ * http://www.gnu.org/ for further details of the GPL. */
+package freenet.support.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+
+import freenet.support.api.Bucket;
+
+/**
+ * A wrapper for a read-only bucket providing for multiple readers. The data
is
+ * only freed when all of the readers have freed it.
+ * @author toad
+ */
+public class MultiReaderBucket {
+
+ private final Bucket bucket;
+
+ // Assume there will be relatively few readers
+ private ArrayList readers;
+
+ private boolean closed;
+
+ public MultiReaderBucket(Bucket underlying) {
+ bucket = underlying;
+ }
+
+ /** Get a reader bucket */
+ public Bucket getReaderBucket() {
+ synchronized(this) {
+ if(closed) return null;
+ Bucket d = new ReaderBucket();
+ readers.add(d);
+ return d;
+ }
+ }
+
+ class ReaderBucket implements Bucket {
+
+ public void free() {
+ synchronized(MultiReaderBucket.this) {
+ readers.remove(this);
+ if(!readers.isEmpty()) return;
+ if(closed) return;
+ closed = true;
+ }
+ bucket.free();
+ }
+
+ public InputStream getInputStream() throws IOException {
+ return bucket.getInputStream();
+ }
+
+ public String getName() {
+ return bucket.getName();
+ }
+
+ public OutputStream getOutputStream() throws IOException {
+ throw new IOException("Read only");
+ }
+
+ public boolean isReadOnly() {
+ return true;
+ }
+
+ public void setReadOnly() {
+ // Already read only
+ }
+
+ public long size() {
+ return bucket.size();
+ }
+
+ public void finalize() {
+ free();
+ }
+
+ }
+
+}
