Author: nextgens
Date: 2008-10-01 21:59:33 +0000 (Wed, 01 Oct 2008)
New Revision: 22907
Added:
trunk/freenet/test/org/apache/
trunk/freenet/test/org/apache/tools/
Modified:
trunk/freenet/src/freenet/client/ArchiveManager.java
trunk/freenet/src/freenet/client/ArchiveStoreContext.java
trunk/freenet/src/freenet/client/Metadata.java
trunk/freenet/src/freenet/client/async/SimpleManifestPutter.java
trunk/freenet/src/freenet/client/async/SingleFileFetcher.java
trunk/freenet/src/freenet/client/async/SingleFileInserter.java
trunk/freenet/src/org/
trunk/freenet/test/org/
Log:
Implements tar.bz2 archive manifests (bug #71)
It doesn't work yet but isn't enabled by default... And it needs to be reviewed
by a client-layer expert ;)
Especially the change to the Metadata constructor! Do we need to set
archiveType when it's a simple manifest?
Modified: trunk/freenet/src/freenet/client/ArchiveManager.java
===================================================================
--- trunk/freenet/src/freenet/client/ArchiveManager.java 2008-10-01
21:01:09 UTC (rev 22906)
+++ trunk/freenet/src/freenet/client/ArchiveManager.java 2008-10-01
21:59:33 UTC (rev 22907)
@@ -18,7 +18,12 @@
import freenet.support.MutableBoolean;
import freenet.support.api.Bucket;
import freenet.support.api.BucketFactory;
+import freenet.support.compress.Bzip2Compressor;
import freenet.support.io.BucketTools;
+import freenet.support.io.Closer;
+import org.apache.tools.bzip2.CBZip2InputStream;
+import org.apache.tools.tar.TarEntry;
+import org.apache.tools.tar.TarInputStream;
/**
* Cache of recently decoded archives:
@@ -34,6 +39,52 @@
public static final String METADATA_NAME = ".metadata";
private static boolean logMINOR;
+ public enum ARCHIVE_TYPE {
+ ZIP((short)0, new String[] { "application/zip",
"application/x-zip" }), /* eventually get rid of ZIP support at some
point */
+ TAR((short)1, new String[] { "application/x-tar" });
+
+ public final short metadataID;
+ public final String[] mimeTypes;
+
+ private ARCHIVE_TYPE(short metadataID, String[] mimeTypes) {
+ this.metadataID = metadataID;
+ this.mimeTypes = mimeTypes;
+ }
+
+ public static boolean isValidMetadataID(short id) {
+ for(ARCHIVE_TYPE current : values())
+ if(id == current.metadataID)
+ return true;
+ return false;
+ }
+
+ /**
+ * Is the given MIME type an archive type that we can deal with?
+ */
+ public static boolean isUsableArchiveType(String type) {
+ for(ARCHIVE_TYPE current : values())
+ for(String ctype : current.mimeTypes)
+ if(ctype.equalsIgnoreCase(type))
+ return true;
+ return false;
+ }
+
+ /** If the given MIME type is an archive type that we can deal
with,
+ * get its archive type number (see the ARCHIVE_ constants in
Metadata).
+ */
+ public static ARCHIVE_TYPE getArchiveType(String type) {
+ for(ARCHIVE_TYPE current : values())
+ for(String ctype : current.mimeTypes)
+ if(ctype.equalsIgnoreCase(type))
+ return current;
+ return null;
+ }
+
+ public final static ARCHIVE_TYPE getDefault() {
+ return ZIP;
+ }
+ }
+
final long maxArchivedFileSize;
// ArchiveHandler's
@@ -154,7 +205,7 @@
/**
* Extract data to cache. Call synchronized on ctx.
* @param key The key the data was fetched from.
- * @param archiveType The archive type. Must be Metadata.ARCHIVE_ZIP.
+ * @param archiveType The archive type. Must be Metadata.ARCHIVE_ZIP |
Metadata.ARCHIVE_TAR.
* @param data The actual data fetched.
* @param archiveContext The context for the whole fetch process.
* @param ctx The ArchiveStoreContext for this key.
@@ -198,9 +249,94 @@
}
if(data.size() > archiveContext.maxArchiveSize)
throw new ArchiveFailureException("Archive too big
("+data.size()+" > "+archiveContext.maxArchiveSize+")!");
- if(archiveType != Metadata.ARCHIVE_ZIP)
+ if(ARCHIVE_TYPE.ZIP.metadataID == archiveType)
+ handleZIPArchive(ctx, key, data, element, callback,
gotElement, throwAtExit);
+ else if(ARCHIVE_TYPE.TAR.metadataID == archiveType)
+ handleTARArchive(ctx, key, data, element, callback,
gotElement, throwAtExit);
+ else
throw new ArchiveFailureException("Unknown or
unsupported archive algorithm "+archiveType);
-
+ }
+
+ private void handleTARArchive(ArchiveStoreContext ctx, FreenetURI key,
Bucket data, String element, ArchiveExtractCallback callback, MutableBoolean
gotElement, boolean throwAtExit) throws ArchiveFailureException,
ArchiveRestartException {
+ if(logMINOR) Logger.minor(this, "Handling a TAR Archive");
+ CBZip2InputStream bz2is = null;
+ TarInputStream tarIS = null;
+ try {
+ bz2is = new CBZip2InputStream(data.getInputStream());
+ tarIS = new TarInputStream(bz2is);
+
+ // MINOR: Assumes the first entry in the tarball is a
directory.
+ TarEntry entry;
+
+ byte[] buf = new byte[32768];
+ HashSet names = new HashSet();
+ boolean gotMetadata = false;
+
+outerTAR: while(true) {
+ entry = tarIS.getNextEntry();
+ if(entry == null) break;
+ if(entry.isDirectory()) continue;
+ String name = entry.getName();
+ if(names.contains(name)) {
+ Logger.error(this, "Duplicate key
"+name+" in archive "+key);
+ continue;
+ }
+ long size = entry.getSize();
+ if(size > maxArchivedFileSize) {
+ addErrorElement(ctx, key, name, "File
too big: "+maxArchivedFileSize+" greater than current archived file size limit
"+maxArchivedFileSize);
+ } else {
+ // Read the element
+ long realLen = 0;
+ Bucket output =
tempBucketFactory.makeBucket(size);
+ OutputStream out =
output.getOutputStream();
+
+ int readBytes;
+ while((readBytes = tarIS.read(buf)) >
0) {
+ out.write(buf, 0, readBytes);
+ readBytes += realLen;
+ if(readBytes >
maxArchivedFileSize) {
+ addErrorElement(ctx,
key, name, "File too big: "+maxArchivedFileSize+" greater than current archived
file size limit "+maxArchivedFileSize);
+ out.close();
+ output.free();
+ continue outerTAR;
+ }
+ }
+
+ out.close();
+ if(name.equals(".metadata"))
+ gotMetadata = true;
+ addStoreElement(ctx, key, name, output,
gotElement, element, callback);
+ names.add(name);
+ trimStoredData();
+ }
+ }
+
+ // If no metadata, generate some
+ if(!gotMetadata) {
+ generateMetadata(ctx, key, names, gotElement,
element, callback);
+ trimStoredData();
+ }
+ if(throwAtExit) throw new
ArchiveRestartException("Archive changed on re-fetch");
+
+ if((!gotElement.value) && element != null)
+ callback.notInArchive();
+
+ } catch (IOException e) {
+ throw new ArchiveFailureException("Error reading
archive: "+e.getMessage(), e);
+ } finally {
+ if(bz2is != null) {
+ try {
+ bz2is.close();
+ } catch (IOException e) {
+ Logger.error(this, "Failed to close
stream: "+e, e);
+ }
+ }
+ Closer.close(tarIS);
+ }
+ }
+
+ private void handleZIPArchive(ArchiveStoreContext ctx, FreenetURI key,
Bucket data, String element, ArchiveExtractCallback callback, MutableBoolean
gotElement, boolean throwAtExit) throws ArchiveFailureException,
ArchiveRestartException {
+ if(logMINOR) Logger.minor(this, "Handling a ZIP Archive");
ZipInputStream zis = null;
try {
zis = new ZipInputStream(data.getInputStream());
@@ -212,7 +348,7 @@
HashSet names = new HashSet();
boolean gotMetadata = false;
-outer: while(true) {
+outerZIP: while(true) {
entry = zis.getNextEntry();
if(entry == null) break;
if(entry.isDirectory()) continue;
@@ -238,7 +374,7 @@
addErrorElement(ctx,
key, name, "File too big: "+maxArchivedFileSize+" greater than current archived
file size limit "+maxArchivedFileSize);
out.close();
output.free();
- continue outer;
+ continue outerZIP;
}
}
@@ -452,21 +588,4 @@
}
}
}
-
- /**
- * Is the given MIME type an archive type that we can deal with?
- */
- public static boolean isUsableArchiveType(String type) {
- return type.equals("application/zip") ||
type.equals("application/x-zip");
- // Update when add new archive types
- }
-
- /** If the given MIME type is an archive type that we can deal with,
- * get its archive type number (see the ARCHIVE_ constants in Metadata).
- */
- public static short getArchiveType(String type) {
- if(type.equals("application/zip") ||
type.equals("application/x-zip"))
- return Metadata.ARCHIVE_ZIP;
- else throw new IllegalArgumentException();
- }
}
Modified: trunk/freenet/src/freenet/client/ArchiveStoreContext.java
===================================================================
--- trunk/freenet/src/freenet/client/ArchiveStoreContext.java 2008-10-01
21:01:09 UTC (rev 22906)
+++ trunk/freenet/src/freenet/client/ArchiveStoreContext.java 2008-10-01
21:59:33 UTC (rev 22907)
@@ -24,7 +24,7 @@
private ArchiveManager manager;
private FreenetURI key;
- private short archiveType;
+ private final short archiveType;
private boolean forceRefetchArchive;
/** Archive size */
private long lastSize = -1;
Modified: trunk/freenet/src/freenet/client/Metadata.java
===================================================================
--- trunk/freenet/src/freenet/client/Metadata.java 2008-10-01 21:01:09 UTC
(rev 22906)
+++ trunk/freenet/src/freenet/client/Metadata.java 2008-10-01 21:59:33 UTC
(rev 22907)
@@ -19,6 +19,7 @@
import freenet.keys.BaseClientKey;
import freenet.keys.ClientCHK;
import freenet.keys.FreenetURI;
+import freenet.client.ArchiveManager.ARCHIVE_TYPE;
import freenet.support.Fields;
import freenet.support.Logger;
import freenet.support.api.Bucket;
@@ -45,8 +46,8 @@
public static final byte SIMPLE_REDIRECT = 0;
static final byte MULTI_LEVEL_METADATA = 1;
static final byte SIMPLE_MANIFEST = 2;
- public static final byte ZIP_MANIFEST = 3;
- public static final byte ZIP_INTERNAL_REDIRECT = 4;
+ public static final byte ARCHIVE_MANIFEST = 3;
+ public static final byte ARCHIVE_INTERNAL_REDIRECT = 4;
// 2 bytes of flags
/** Is a splitfile */
@@ -70,15 +71,15 @@
// static final short FLAGS_SPLIT_USE_LENGTHS = 64; FIXME not supported,
reassign to something else if we need a new flag
static final short FLAGS_COMPRESSED = 128;
- /** Container archive type */
+ /** Container archive type
+ * @see ARCHIVE_TYPE
+ */
short archiveType;
- static final short ARCHIVE_ZIP = 0;
- static final short ARCHIVE_TAR = 1; // FIXME for future use
- /** Compressed splitfile codec */
+ /** Compressed splitfile codec
+ * @see COMPRESSOR_TYPE
+ */
short compressionCodec = -1;
- public static final short COMPRESS_GZIP =
COMPRESSOR_TYPE.GZIP.metadataID;
- public static final short COMPRESS_BZIP2 =
COMPRESSOR_TYPE.BZIP2.metadataID;
/** The length of the splitfile */
long dataLength;
@@ -183,7 +184,7 @@
boolean compressed = false;
if((documentType == SIMPLE_REDIRECT) || (documentType ==
MULTI_LEVEL_METADATA)
- || (documentType == ZIP_MANIFEST) ||
(documentType == ZIP_INTERNAL_REDIRECT)) {
+ || (documentType == ARCHIVE_MANIFEST) ||
(documentType == ARCHIVE_INTERNAL_REDIRECT)) {
short flags = dis.readShort();
splitfile = (flags & FLAGS_SPLITFILE) ==
FLAGS_SPLITFILE;
dbr = (flags & FLAGS_DBR) == FLAGS_DBR;
@@ -194,10 +195,10 @@
compressed = (flags & FLAGS_COMPRESSED) ==
FLAGS_COMPRESSED;
}
- if(documentType == ZIP_MANIFEST) {
- if(logMINOR) Logger.minor(this, "Zip manifest");
+ if(documentType == ARCHIVE_MANIFEST) {
+ if(logMINOR) Logger.minor(this, "Archive manifest");
archiveType = dis.readShort();
- if(archiveType != ARCHIVE_ZIP)
+ if(!ARCHIVE_TYPE.isValidMetadataID(archiveType))
throw new MetadataParseException("Unrecognized
archive type "+archiveType);
}
@@ -268,7 +269,7 @@
clientMetadata = new ClientMetadata(mimeType);
- if((!splitfile) && ((documentType == SIMPLE_REDIRECT) ||
(documentType == ZIP_MANIFEST))) {
+ if((!splitfile) && ((documentType == SIMPLE_REDIRECT) ||
(documentType == ARCHIVE_MANIFEST))) {
simpleRedirectKey = readKey(dis);
} else if(splitfile) {
splitfileAlgorithm = dis.readShort();
@@ -345,13 +346,13 @@
if(logMINOR) Logger.minor(this, "End of manifest"); //
Make it easy to search for it!
}
- if(documentType == ZIP_INTERNAL_REDIRECT) {
+ if(documentType == ARCHIVE_INTERNAL_REDIRECT) {
int len = dis.readShort();
if(logMINOR) Logger.minor(this, "Reading zip internal
redirect length "+len);
byte[] buf = new byte[len];
dis.readFully(buf);
nameInArchive = new String(buf, "UTF-8");
- if(logMINOR) Logger.minor(this, "Zip internal redirect:
"+nameInArchive+" ("+len+ ')');
+ if(logMINOR) Logger.minor(this, "Archive internal
redirect: "+nameInArchive+" ("+len+ ')');
}
}
@@ -470,7 +471,7 @@
Metadata target;
if(o instanceof String) {
// Zip internal redirect
- target = new Metadata(ZIP_INTERNAL_REDIRECT,
prefix+key, new ClientMetadata(DefaultMIMETypes.guessMIMEType(key, false)));
+ target = new
Metadata(ARCHIVE_INTERNAL_REDIRECT, prefix+key, new
ClientMetadata(DefaultMIMETypes.guessMIMEType(key, false)));
} else if(o instanceof HashMap) {
target = new Metadata((HashMap)o,
prefix+key+"/");
} else throw new IllegalArgumentException("Not String
nor HashMap: "+o);
@@ -486,7 +487,7 @@
* the archive to read from.
*/
public Metadata(byte docType, String arg, ClientMetadata cm) {
- if(docType == ZIP_INTERNAL_REDIRECT) {
+ if(docType == ARCHIVE_INTERNAL_REDIRECT) {
documentType = docType;
// Determine MIME type
this.clientMetadata = cm;
@@ -504,8 +505,9 @@
* @param cm The client metadata, if any.
*/
public Metadata(byte docType, FreenetURI uri, ClientMetadata cm) {
- if((docType == SIMPLE_REDIRECT) || (docType == ZIP_MANIFEST)) {
+ if((docType == SIMPLE_REDIRECT) || (docType ==
ARCHIVE_MANIFEST)) {
documentType = docType;
+ archiveType = ARCHIVE_TYPE.getDefault().metadataID;
clientMetadata = cm;
if((cm != null) && !cm.isTrivial()) {
setMIMEType(cm.getMIMEType());
@@ -526,7 +528,7 @@
documentType = MULTI_LEVEL_METADATA;
else {
if(insertAsArchiveManifest)
- documentType = ZIP_MANIFEST;
+ documentType = ARCHIVE_MANIFEST;
else documentType = SIMPLE_REDIRECT;
}
splitfile = true;
@@ -668,7 +670,7 @@
public boolean isSingleFileRedirect() {
return (((!splitfile) &&
(documentType == SIMPLE_REDIRECT)) ||
(documentType == MULTI_LEVEL_METADATA) ||
- (documentType == ZIP_MANIFEST));
+ (documentType == ARCHIVE_MANIFEST));
}
/**
@@ -682,7 +684,7 @@
* Is this a ZIP manifest?
*/
public boolean isArchiveManifest() {
- return documentType == ZIP_MANIFEST;
+ return documentType == ARCHIVE_MANIFEST;
}
/**
@@ -690,7 +692,7 @@
* @return
*/
public boolean isArchiveInternalRedirect() {
- return documentType == ZIP_INTERNAL_REDIRECT;
+ return documentType == ARCHIVE_INTERNAL_REDIRECT;
}
/**
@@ -743,7 +745,7 @@
dos.writeShort(0); // version
dos.writeByte(documentType);
if((documentType == SIMPLE_REDIRECT) || (documentType ==
MULTI_LEVEL_METADATA)
- || (documentType == ZIP_MANIFEST) ||
(documentType == ZIP_INTERNAL_REDIRECT)) {
+ || (documentType == ARCHIVE_MANIFEST) ||
(documentType == ARCHIVE_INTERNAL_REDIRECT)) {
short flags = 0;
if(splitfile) flags |= FLAGS_SPLITFILE;
if(dbr) flags |= FLAGS_DBR;
@@ -755,7 +757,7 @@
dos.writeShort(flags);
}
- if(documentType == ZIP_MANIFEST) {
+ if(documentType == ARCHIVE_MANIFEST) {
dos.writeShort(archiveType);
}
@@ -789,7 +791,7 @@
if(extraMetadata)
throw new UnsupportedOperationException("No extra
metadata support yet");
- if((!splitfile) && ((documentType == SIMPLE_REDIRECT) ||
(documentType == ZIP_MANIFEST))) {
+ if((!splitfile) && ((documentType == SIMPLE_REDIRECT) ||
(documentType == ARCHIVE_MANIFEST))) {
writeKey(dos, simpleRedirectKey);
} else if(splitfile) {
dos.writeShort(splitfileAlgorithm);
@@ -850,7 +852,7 @@
}
}
- if(documentType == ZIP_INTERNAL_REDIRECT) {
+ if(documentType == ARCHIVE_INTERNAL_REDIRECT) {
byte[] data = nameInArchive.getBytes("UTF-8");
if(data.length > Short.MAX_VALUE) throw new
IllegalArgumentException("Zip internal redirect name too long");
dos.writeShort(data.length);
@@ -911,9 +913,9 @@
}
public void setArchiveManifest() {
- archiveType =
ArchiveManager.getArchiveType(clientMetadata.getMIMEType());
+ archiveType =
ARCHIVE_TYPE.getArchiveType(clientMetadata.getMIMEType()).metadataID;
clientMetadata.clear();
- documentType = ZIP_MANIFEST;
+ documentType = ARCHIVE_MANIFEST;
}
public String getMIMEType() {
Modified: trunk/freenet/src/freenet/client/async/SimpleManifestPutter.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SimpleManifestPutter.java
2008-10-01 21:01:09 UTC (rev 22906)
+++ trunk/freenet/src/freenet/client/async/SimpleManifestPutter.java
2008-10-01 21:59:33 UTC (rev 22907)
@@ -1,5 +1,6 @@
package freenet.client.async;
+import freenet.client.ArchiveManager.ARCHIVE_TYPE;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
@@ -9,7 +10,6 @@
import java.util.LinkedList;
import java.util.Vector;
import java.util.zip.ZipEntry;
-import java.util.zip.ZipException;
import java.util.zip.ZipOutputStream;
import freenet.client.ClientMetadata;
@@ -25,6 +25,9 @@
import freenet.support.Logger;
import freenet.support.api.Bucket;
import freenet.support.io.BucketTools;
+import org.apache.tools.bzip2.CBZip2OutputStream;
+import org.apache.tools.tar.TarEntry;
+import org.apache.tools.tar.TarOutputStream;
public class SimpleManifestPutter extends BaseClientPutter implements
PutCompletionCallback {
// Only implements PutCompletionCallback for the final metadata insert
@@ -51,12 +54,12 @@
origSFI = null;
}
- protected PutHandler(final SimpleManifestPutter smp, String
name, String targetInZip, ClientMetadata cm, Bucket data) {
+ protected PutHandler(final SimpleManifestPutter smp, String
name, String targetInArchive, ClientMetadata cm, Bucket data) {
super(smp.getPriorityClass(), smp.chkScheduler,
smp.sskScheduler, smp.client);
this.cm = cm;
this.data = data;
- this.targetInZip = targetInZip;
- Metadata m = new
Metadata(Metadata.ZIP_INTERNAL_REDIRECT, targetInZip, cm);
+ this.targetInArchive = targetInArchive;
+ Metadata m = new
Metadata(Metadata.ARCHIVE_INTERNAL_REDIRECT, targetInArchive, cm);
metadata = m;
origSFI = null;
}
@@ -64,7 +67,7 @@
private SingleFileInserter origSFI;
private ClientMetadata cm;
private Metadata metadata;
- private String targetInZip;
+ private String targetInArchive;
private final Bucket data;
public void start() throws InsertException {
@@ -222,7 +225,7 @@
private final static String[] defaultDefaultNames =
new String[] { "index.html", "index.htm", "default.html",
"default.htm" };
private int bytesOnZip;
- private LinkedList elementsToPutInZip;
+ private LinkedList<PutHandler> elementsToPutInArchive;
private boolean fetchable;
private final boolean earlyEncode;
@@ -244,7 +247,7 @@
waitingForBlockSets = new HashSet();
metadataPuttersByMetadata = new HashMap();
metadataPuttersUnfetchable = new HashMap();
- elementsToPutInZip = new LinkedList();
+ elementsToPutInArchive = new LinkedList();
makePutHandlers(manifestElements, putHandlersByName);
checkZips();
}
@@ -324,7 +327,7 @@
bytesOnZip += sz;
// Put it in the zip.
ph = new PutHandler(this, name,
ZipPrefix+element.fullName, cm, data);
- elementsToPutInZip.addLast(ph);
+
elementsToPutInArchive.addLast(ph);
numberOfFiles++;
totalSize += data.size();
} else {
@@ -415,52 +418,26 @@
InsertBlock block;
boolean isMetadata = true;
boolean insertAsArchiveManifest = false;
- if(!(elementsToPutInZip.isEmpty())) {
- // There is a zip to insert.
+ if(!(elementsToPutInArchive.isEmpty())) {
+ // There is an archive to insert.
// We want to include the metadata.
// We have the metadata, fortunately enough, because
everything has been resolve()d.
- // So all we need to do is create the actual ZIP.
- try {
+ // So all we need to do is create the actual archive.
+ try {
+ Bucket outputBucket =
ctx.bf.makeBucket(baseMetadata.dataLength());
+ // TODO: try both ? - maybe not worth it
+ String mimeType = (ARCHIVE_TYPE.getDefault() ==
ARCHIVE_TYPE.TAR ?
+ createTarBucket(bucket, outputBucket) :
+ createZipBucket(bucket, outputBucket));
- // FIXME support formats other than .zip.
- // Only the *decoding* is generic at present.
+ // Now we have to insert the Archive we have
generated.
- Bucket zipBucket =
ctx.bf.makeBucket(baseMetadata.dataLength());
- OutputStream os = new
BufferedOutputStream(zipBucket.getOutputStream());
- ZipOutputStream zos = new ZipOutputStream(os);
- ZipEntry ze;
-
- for(Iterator
i=elementsToPutInZip.iterator();i.hasNext();) {
- PutHandler ph = (PutHandler) i.next();
- ze = new ZipEntry(ph.targetInZip);
- ze.setTime(0);
- zos.putNextEntry(ze);
- BucketTools.copyTo(ph.data, zos,
ph.data.size());
- }
-
- // Add .metadata - after the rest.
- ze = new ZipEntry(".metadata");
- ze.setTime(0); // -1 = now, 0 = 1970.
- zos.putNextEntry(ze);
- BucketTools.copyTo(bucket, zos, bucket.size());
-
- zos.closeEntry();
- // Both finish() and close() are necessary.
- zos.finish();
- zos.flush();
- zos.close();
-
- // Now we have to insert the ZIP.
-
// Can we just insert it, and not bother with a
redirect to it?
// Thereby exploiting implicit manifest
support, which will pick up on .metadata??
// We ought to be able to !!
- block = new InsertBlock(zipBucket, new
ClientMetadata("application/zip"), targetURI);
+ block = new InsertBlock(outputBucket, new
ClientMetadata(mimeType), targetURI);
isMetadata = false;
insertAsArchiveManifest = true;
- } catch (ZipException e) {
- fail(new
InsertException(InsertException.INTERNAL_ERROR, e, null));
- return;
} catch (IOException e) {
fail(new
InsertException(InsertException.BUCKET_ERROR, e, null));
return;
@@ -479,6 +456,72 @@
}
}
+ private String createTarBucket(Bucket inputBucket, Bucket outputBucket)
throws IOException {
+ if(logMINOR) Logger.minor(this, "Create a TAR Bucket");
+
+ OutputStream os = new
BufferedOutputStream(outputBucket.getOutputStream());
+ CBZip2OutputStream bz2OS = new CBZip2OutputStream(os);
+ TarOutputStream tarOS = new TarOutputStream(bz2OS);
+ TarEntry ze;
+
+ for(PutHandler ph : elementsToPutInArchive) {
+ ze = new TarEntry(ph.targetInArchive);
+ ze.setModTime(0);
+ long size = ph.data.size();
+ ze.setSize(size);
+ tarOS.putNextEntry(ze);
+ BucketTools.copyTo(ph.data, tarOS, size);
+ tarOS.closeEntry();
+ }
+
+ // Add .metadata - after the rest.
+ ze = new TarEntry(".metadata");
+ ze.setModTime(0); // -1 = now, 0 = 1970.
+ long size = inputBucket.size();
+ ze.setSize(size);
+ tarOS.putNextEntry(ze);
+ BucketTools.copyTo(inputBucket, tarOS, size);
+
+ tarOS.closeEntry();
+ // Both finish() and close() are necessary.
+ tarOS.finish();
+ tarOS.flush();
+ tarOS.close();
+
+ return ARCHIVE_TYPE.TAR.mimeTypes[0];
+ }
+
+ private String createZipBucket(Bucket inputBucket, Bucket outputBucket)
throws IOException {
+ if(logMINOR) Logger.minor(this, "Create a ZIP Bucket");
+
+ OutputStream os = new
BufferedOutputStream(outputBucket.getOutputStream());
+ ZipOutputStream zos = new ZipOutputStream(os);
+ ZipEntry ze;
+
+ for(Iterator i = elementsToPutInArchive.iterator();
i.hasNext();) {
+ PutHandler ph = (PutHandler) i.next();
+ ze = new ZipEntry(ph.targetInArchive);
+ ze.setTime(0);
+ zos.putNextEntry(ze);
+ BucketTools.copyTo(ph.data, zos, ph.data.size());
+ zos.closeEntry();
+ }
+
+ // Add .metadata - after the rest.
+ ze = new ZipEntry(".metadata");
+ ze.setTime(0); // -1 = now, 0 = 1970.
+ zos.putNextEntry(ze);
+ BucketTools.copyTo(inputBucket, zos, inputBucket.size());
+
+ zos.closeEntry();
+ // Both finish() and close() are necessary.
+ zos.finish();
+ zos.flush();
+ zos.close();
+
+ return ARCHIVE_TYPE.ZIP.mimeTypes[0];
+ }
+
private boolean resolve(MetadataUnresolvedException e) throws
InsertException, IOException {
Metadata[] metas = e.mustResolve;
boolean mustWait = false;
Modified: trunk/freenet/src/freenet/client/async/SingleFileFetcher.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SingleFileFetcher.java
2008-10-01 21:01:09 UTC (rev 22906)
+++ trunk/freenet/src/freenet/client/async/SingleFileFetcher.java
2008-10-01 21:59:33 UTC (rev 22907)
@@ -162,7 +162,7 @@
metadata = Metadata.construct(data);
wrapHandleMetadata(false);
} catch (MetadataParseException e) {
- onFailure(new FetchException(e), sched);
+ onFailure(new
FetchException(FetchException.INVALID_METADATA, e), sched);
return;
} catch (IOException e) {
// Bucket error?
@@ -255,11 +255,11 @@
metadata = metadata.getDocument(name);
thisKey = thisKey.pushMetaString(name);
if(metadata == null)
- throw new
FetchException(FetchException.NOT_IN_ARCHIVE);
+ throw new
FetchException(FetchException.NOT_IN_ARCHIVE, "can't find "+name);
}
continue; // loop
} else if(metadata.isArchiveManifest()) {
- if(logMINOR) Logger.minor(this, "Is archive
manifest");
+ if(logMINOR) Logger.minor(this, "Is archive
manifest (type="+metadata.getArchiveType()+')');
if(metaStrings.isEmpty() &&
ctx.returnZIPManifests) {
// Just return the archive, whole.
metadata.setSimpleRedirect();
@@ -396,7 +396,7 @@
if(mime != null) rcb.onExpectedMIME(mime);
String mimeType =
clientMetadata.getMIMETypeNoParams();
- if(mimeType != null &&
ArchiveManager.isUsableArchiveType(mimeType) && metaStrings.size() > 0) {
+ if(mimeType != null &&
ArchiveManager.ARCHIVE_TYPE.isUsableArchiveType(mimeType) && metaStrings.size()
> 0) {
// Looks like an implicit archive,
handle as such
metadata.setArchiveManifest();
// Pick up MIME type from inside archive
@@ -461,7 +461,7 @@
if(mime != null) rcb.onExpectedMIME(mime);
String mimeType =
clientMetadata.getMIMETypeNoParams();
- if(mimeType != null &&
ArchiveManager.isUsableArchiveType(mimeType) && metaStrings.size() > 0) {
+ if(mimeType != null &&
ArchiveManager.ARCHIVE_TYPE.isUsableArchiveType(mimeType) && metaStrings.size()
> 0) {
// Looks like an implicit archive,
handle as such
metadata.setArchiveManifest();
// Pick up MIME type from inside archive
@@ -573,7 +573,7 @@
try {
handleMetadata();
} catch (MetadataParseException e) {
- onFailure(new FetchException(e), sched);
+ onFailure(new
FetchException(FetchException.INVALID_METADATA, e), sched);
} catch (FetchException e) {
if(notFinalizedSize)
e.setNotFinalizedSize();
Modified: trunk/freenet/src/freenet/client/async/SingleFileInserter.java
===================================================================
--- trunk/freenet/src/freenet/client/async/SingleFileInserter.java
2008-10-01 21:01:09 UTC (rev 22906)
+++ trunk/freenet/src/freenet/client/async/SingleFileInserter.java
2008-10-01 21:59:33 UTC (rev 22907)
@@ -294,7 +294,7 @@
}
private Metadata makeMetadata(FreenetURI uri) {
- Metadata meta = new Metadata(insertAsArchiveManifest ?
Metadata.ZIP_MANIFEST : Metadata.SIMPLE_REDIRECT, uri, block.clientMetadata);
+ Metadata meta = new Metadata(insertAsArchiveManifest ?
Metadata.ARCHIVE_MANIFEST : Metadata.SIMPLE_REDIRECT, uri,
block.clientMetadata);
if(targetFilename != null) {
HashMap hm = new HashMap();
hm.put(targetFilename, meta);
Property changes on: trunk/freenet/src/org
___________________________________________________________________
Name: svn:externals
- apache/tools/bzip2
http://svn.apache.org/repos/asf/ant/core/tags/ANT_171/src/main/org/apache/tools/bzip2/
+ apache/tools/bzip2
http://svn.apache.org/repos/asf/ant/core/tags/ANT_171/src/main/org/apache/tools/bzip2/
apache/tools/tar
http://svn.apache.org/repos/asf/ant/core/tags/ANT_171/src/main/org/apache/tools/tar/
Property changes on: trunk/freenet/test/org
___________________________________________________________________
Name: svn:externals
+ apache/tools/tar
http://svn.apache.org/repos/asf/ant/core/tags/ANT_171/src/tests/junit/org/apache/tools/tar/
