Author: toad
Date: 2006-01-14 20:24:24 +0000 (Sat, 14 Jan 2006)
New Revision: 7857
Added:
trunk/freenet/src/freenet/client/MultiFileInserter.java
Modified:
trunk/freenet/src/freenet/client/BlockInserter.java
trunk/freenet/src/freenet/client/FileInserter.java
trunk/freenet/src/freenet/client/HighLevelSimpleClient.java
trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
trunk/freenet/src/freenet/client/InserterException.java
trunk/freenet/src/freenet/client/Metadata.java
trunk/freenet/src/freenet/client/SplitInserter.java
trunk/freenet/src/freenet/clients/http/ToadletContextImpl.java
trunk/freenet/src/freenet/node/TextModeClientInterface.java
trunk/freenet/src/freenet/node/Version.java
Log:
353:
Revamp PUTDIR. It is now part of freenet/client/.
- Much faster (we insert 5 files at a time)
- More efficient (we aggregate the metadata)
- Will detect {index|default}.{htm|html} as default
- Can be exposed to FCP (not yet obviously), and later support containers etc
Also minor fproxy changes.
Modified: trunk/freenet/src/freenet/client/BlockInserter.java
===================================================================
--- trunk/freenet/src/freenet/client/BlockInserter.java 2006-01-14 17:36:25 UTC
(rev 7856)
+++ trunk/freenet/src/freenet/client/BlockInserter.java 2006-01-14 20:24:24 UTC
(rev 7857)
@@ -73,8 +73,8 @@
FileInserter inserter = new FileInserter(ctx);
try {
if(uri == null && !getCHKOnly)
- uri = inserter.run(block, false, true, true);
- uri = inserter.run(block, false, getCHKOnly, true);
+ uri = inserter.run(block, false, true, true,
null);
+ uri = inserter.run(block, false, getCHKOnly, true,
null);
succeeded = true;
tracker.success(this);
} catch (InserterException e) {
Modified: trunk/freenet/src/freenet/client/FileInserter.java
===================================================================
--- trunk/freenet/src/freenet/client/FileInserter.java 2006-01-14 17:36:25 UTC
(rev 7856)
+++ trunk/freenet/src/freenet/client/FileInserter.java 2006-01-14 20:24:24 UTC
(rev 7857)
@@ -1,19 +1,17 @@
package freenet.client;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
-import freenet.client.events.BlockInsertErrorEvent;
import freenet.client.events.SimpleBlockPutEvent;
import freenet.keys.CHKBlock;
import freenet.keys.CHKEncodeException;
import freenet.keys.ClientCHKBlock;
-import freenet.keys.ClientSSK;
import freenet.keys.ClientSSKBlock;
import freenet.keys.FreenetURI;
import freenet.keys.InsertableClientSSK;
import freenet.keys.NodeCHK;
-import freenet.keys.NodeSSK;
import freenet.keys.SSKBlock;
import freenet.keys.SSKEncodeException;
import freenet.node.LowLevelPutException;
@@ -37,11 +35,15 @@
/**
* Do an insert.
* @param block The data to insert.
- * @param localOnly
+ * @param localOnly
+ * @param returnMetadata If not null, return the metadata in this
bucket, rather
+ * than inserting it; return the *data* CHK only. This is used by e.g.
+ * MultiFileInserter, where we will aggregate the metadata elsewhere.
+ * Only supported on CHKs.
* @return The URI of the inserted data.
* @throws InserterException
*/
- public FreenetURI run(InsertBlock block, boolean metadata, boolean
getCHKOnly, boolean noRetries) throws InserterException {
+ public FreenetURI run(InsertBlock block, boolean metadata, boolean
getCHKOnly, boolean noRetries, Bucket returnMetadata) throws InserterException {
if(block.data == null)
throw new NullPointerException();
if(block.desiredURI.getKeyType().equalsIgnoreCase("CHK")) {
@@ -149,7 +151,7 @@
if(isSSK) {
// Insert as CHK
// Create metadata pointing to it (include the
clientMetadata if there is any).
- FreenetURI uri = run(new InsertBlock(block.data, new
ClientMetadata(), FreenetURI.EMPTY_CHK_URI), metadata, getCHKOnly, noRetries);
+ FreenetURI uri = run(new InsertBlock(block.data, new
ClientMetadata(), FreenetURI.EMPTY_CHK_URI), metadata, getCHKOnly, noRetries,
null);
Metadata m = new Metadata(Metadata.SIMPLE_REDIRECT,
uri, block.clientMetadata);
Bucket bucket;
try {
@@ -157,7 +159,7 @@
} catch (IOException e) {
throw new
InserterException(InserterException.INTERNAL_ERROR, e, isk.getURI());
}
- return run(new InsertBlock(bucket, new
ClientMetadata(), block.desiredURI), metadata, getCHKOnly, noRetries);
+ return run(new InsertBlock(bucket, new
ClientMetadata(), block.desiredURI), metadata, getCHKOnly, noRetries, null);
}
if(data.size() <= NodeCHK.BLOCK_SIZE) {
@@ -175,11 +177,11 @@
Logger.error(this, "Unexpected error: "+e, e);
throw new
InserterException(InserterException.INTERNAL_ERROR, null);
}
- return simplePutCHK(chk, block.clientMetadata,
getCHKOnly, noRetries);
+ return simplePutCHK(chk, block.clientMetadata,
getCHKOnly, noRetries, returnMetadata);
}
// Too big, encode to a splitfile
- SplitInserter splitInsert = new SplitInserter(data,
block.clientMetadata, bestCodec, ctx.splitfileAlgorithm, ctx, this,
NodeCHK.BLOCK_SIZE, getCHKOnly, metadata);
+ SplitInserter splitInsert = new SplitInserter(data,
block.clientMetadata, bestCodec, ctx.splitfileAlgorithm, ctx, this,
NodeCHK.BLOCK_SIZE, getCHKOnly, metadata, returnMetadata);
return splitInsert.run();
}
@@ -188,10 +190,13 @@
* @param chk The data encoded into a single CHK.
* @param clientMetadata The client metadata. If this is non-trivial,
we will have to
* create a redirect document just to put the metadata in.
+ * @param returnMetadata If not null, return the metadata in this
bucket, rather
+ * than inserting it; return the *data* CHK only. This is used by e.g.
+ * MultiFileInserter, where we will aggregate the metadata elsewhere.
* @return The URI of the resulting CHK.
* @throws InserterException If there was an error inserting the block.
*/
- private FreenetURI simplePutCHK(ClientCHKBlock chk, ClientMetadata
clientMetadata, boolean getCHKOnly, boolean noRetries) throws InserterException
{
+ private FreenetURI simplePutCHK(ClientCHKBlock chk, ClientMetadata
clientMetadata, boolean getCHKOnly, boolean noRetries, Bucket returnMetadata)
throws InserterException {
LowLevelPutException le = null;
int rnfs = 0;
for(int i=0;i<=ctx.maxInsertRetries;i++) {
@@ -224,11 +229,22 @@
if(clientMetadata == null || clientMetadata.isTrivial())
// Don't need a redirect for the metadata
- uri = chk.getClientKey().getURI();
+ uri = chk.getClientKey().getURI();
else {
// Do need a redirect for the metadata
Metadata metadata = new
Metadata(Metadata.SIMPLE_REDIRECT, chk.getClientKey().getURI(), clientMetadata);
- uri = putMetadataCHK(metadata, getCHKOnly, noRetries);
+ if(returnMetadata != null) {
+ uri = chk.getClientKey().getURI();
+ try {
+ DataOutputStream dos = new
DataOutputStream(returnMetadata.getOutputStream());
+ metadata.writeTo(dos);
+ dos.close();
+ } catch (IOException e) {
+ throw new
InserterException(InserterException.BUCKET_ERROR);
+ }
+ } else {
+ uri = putMetadataCHK(metadata, getCHKOnly,
noRetries);
+ }
}
if(le != null)
@@ -306,6 +322,6 @@
throw new
InserterException(InserterException.BUCKET_ERROR, null);
}
InsertBlock block = new InsertBlock(bucket, null,
FreenetURI.EMPTY_CHK_URI);
- return run(block, true, getCHKOnly, noRetries);
+ return run(block, true, getCHKOnly, noRetries, null);
}
}
Modified: trunk/freenet/src/freenet/client/HighLevelSimpleClient.java
===================================================================
--- trunk/freenet/src/freenet/client/HighLevelSimpleClient.java 2006-01-14
17:36:25 UTC (rev 7856)
+++ trunk/freenet/src/freenet/client/HighLevelSimpleClient.java 2006-01-14
20:24:24 UTC (rev 7857)
@@ -38,7 +38,7 @@
/**
* Blocking insert of multiple files as a manifest (or zip manifest,
etc).
*/
- public FreenetURI insertManifest(FreenetURI insertURI, HashMap
bucketsByName) throws InserterException;
+ public FreenetURI insertManifest(FreenetURI insertURI, HashMap
bucketsByName, String defaultName) throws InserterException;
/**
* Add a ClientEventListener.
Modified: trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
===================================================================
--- trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
2006-01-14 17:36:25 UTC (rev 7856)
+++ trunk/freenet/src/freenet/client/HighLevelSimpleClientImpl.java
2006-01-14 20:24:24 UTC (rev 7857)
@@ -109,14 +109,14 @@
InserterContext context = new InserterContext(client,
bucketFactory, random, INSERT_RETRIES, CONSECUTIVE_RNFS_ASSUME_SUCCESS,
SPLITFILE_INSERT_THREADS,
SPLITFILE_BLOCKS_PER_SEGMENT, SPLITFILE_CHECK_BLOCKS_PER_SEGMENT,
globalEventProducer, insertStarter, cacheLocalRequests);
FileInserter i = new FileInserter(context);
- return i.run(insert, false, getCHKOnly, false);
+ return i.run(insert, false, getCHKOnly, false, null);
}
public FreenetURI insert(InsertBlock insert, boolean getCHKOnly,
boolean metadata) throws InserterException {
InserterContext context = new InserterContext(client,
bucketFactory, random, INSERT_RETRIES, CONSECUTIVE_RNFS_ASSUME_SUCCESS,
SPLITFILE_INSERT_THREADS,
SPLITFILE_BLOCKS_PER_SEGMENT, SPLITFILE_CHECK_BLOCKS_PER_SEGMENT,
globalEventProducer, insertStarter, cacheLocalRequests);
FileInserter i = new FileInserter(context);
- return i.run(insert, metadata, getCHKOnly, false);
+ return i.run(insert, metadata, getCHKOnly, false, null);
}
public FreenetURI insertRedirect(FreenetURI insertURI, FreenetURI
targetURI) throws InserterException {
@@ -133,12 +133,14 @@
InserterContext context = new InserterContext(client,
bucketFactory, random, INSERT_RETRIES, CONSECUTIVE_RNFS_ASSUME_SUCCESS,
SPLITFILE_INSERT_THREADS,
SPLITFILE_BLOCKS_PER_SEGMENT, SPLITFILE_CHECK_BLOCKS_PER_SEGMENT,
globalEventProducer, insertStarter, cacheLocalRequests);
FileInserter i = new FileInserter(context);
- return i.run(block, true, false, false);
+ return i.run(block, true, false, false, null);
}
- public FreenetURI insertManifest(FreenetURI insertURI, HashMap
bucketsByName) throws InserterException {
- // FIXME
- throw new UnsupportedOperationException();
+ public FreenetURI insertManifest(FreenetURI insertURI, HashMap
bucketsByName, String defaultName) throws InserterException {
+ InserterContext context = new InserterContext(client,
bucketFactory, random, INSERT_RETRIES, CONSECUTIVE_RNFS_ASSUME_SUCCESS,
+ SPLITFILE_INSERT_THREADS,
SPLITFILE_BLOCKS_PER_SEGMENT, SPLITFILE_CHECK_BLOCKS_PER_SEGMENT,
globalEventProducer, insertStarter, cacheLocalRequests);
+ MultiFileInserter mfi = new MultiFileInserter(insertURI,
bucketsByName, context, defaultName);
+ return mfi.run();
}
public void addGlobalHook(ClientEventListener listener) {
Modified: trunk/freenet/src/freenet/client/InserterException.java
===================================================================
--- trunk/freenet/src/freenet/client/InserterException.java 2006-01-14
17:36:25 UTC (rev 7856)
+++ trunk/freenet/src/freenet/client/InserterException.java 2006-01-14
20:24:24 UTC (rev 7857)
@@ -51,6 +51,7 @@
}
public InserterException(int mode) {
+ super(getMessage(mode));
this.mode = mode;
this.errorCodes = null;
this.uri = null;
Modified: trunk/freenet/src/freenet/client/Metadata.java
===================================================================
--- trunk/freenet/src/freenet/client/Metadata.java 2006-01-14 17:36:25 UTC
(rev 7856)
+++ trunk/freenet/src/freenet/client/Metadata.java 2006-01-14 20:24:24 UTC
(rev 7857)
@@ -292,7 +292,35 @@
ret.addRedirectionManifest(dir);
return ret;
}
+
+ /**
+ * Create a Metadata object and add manifest entries from the given
array. */
+ public static Metadata mkRedirectionManifestWithMetadata(HashMap dir) {
+ Metadata ret = new Metadata();
+ ret.addRedirectionManifestWithMetadata(dir);
+ return ret;
+ }
+ private void addRedirectionManifestWithMetadata(HashMap dir) {
+ // Simple manifest - contains actual redirects.
+ // Not zip manifest, which is basically a redirect.
+ documentType = SIMPLE_MANIFEST;
+ noMIME = true;
+ //mimeType = null;
+ //clientMetadata = new ClientMetadata(null);
+ manifestEntries = new HashMap();
+ int count = 0;
+ for(Iterator i = dir.keySet().iterator();i.hasNext();) {
+ String key = (String) i.next();
+ count++;
+ byte[] data = (byte[]) dir.get(key);
+ if(data == null)
+ throw new NullPointerException();
+ manifestEntries.put(key, data);
+ }
+ manifestEntryCount = count;
+ }
+
/**
* Create a Metadata object for an archive which does not have its own
* metadata.
Added: trunk/freenet/src/freenet/client/MultiFileInserter.java
===================================================================
--- trunk/freenet/src/freenet/client/MultiFileInserter.java 2006-01-14
17:36:25 UTC (rev 7856)
+++ trunk/freenet/src/freenet/client/MultiFileInserter.java 2006-01-14
20:24:24 UTC (rev 7857)
@@ -0,0 +1,191 @@
+package freenet.client;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import freenet.keys.FreenetURI;
+import freenet.support.ArrayBucket;
+import freenet.support.Bucket;
+import freenet.support.BucketTools;
+import freenet.support.Logger;
+
+public class MultiFileInserter {
+
+ public class MFInserter implements Runnable {
+
+ final int num;
+
+ MFInserter(int x) {
+ num = x;
+ }
+
+ public void run() {
+ try {
+ while(true) {
+ String name = null;
+ Bucket data = null;
+ synchronized(bucketsByName) {
+ if(bucketsByName.isEmpty()) break;
+ name = (String)
bucketsByName.keySet().iterator().next();
+ data = (Bucket)
bucketsByName.remove(name);
+ }
+ String mimeType =
DefaultMIMETypes.guessMIMEType(name);
+ Logger.minor(this, "Name: "+name+"\nBucket
size: "+data.size()+"\nGuessed MIME type: "+mimeType);
+ byte[] metaByteArray;
+ try {
+ metaByteArray = getMetadata(name, data,
mimeType);
+ } catch (InserterException e) {
+ Logger.normal(this, "Error inserting
"+name+": "+e.getMessage());
+ errorCodes.inc(e.getMode());
+ synchronized(this) {
+ errors++;
+ }
+ continue;
+ } catch (Throwable t) {
+ Logger.error(this, "Caught "+t);
+
errorCodes.inc(InserterException.INTERNAL_ERROR);
+ synchronized(this) {
+ errors++;
+ }
+ continue;
+ }
+ if(metaByteArray != null) {
+ synchronized(namesToMetadataByteArrays)
{
+
namesToMetadataByteArrays.put(name, metaByteArray);
+ }
+ Logger.minor(this, "Inserted "+name);
+ } else {
+ Logger.normal(this, "Insert failed:
"+name);
+ }
+
+ }
+ } finally {
+ synchronized(MultiFileInserter.this) {
+ finished[num] = true;
+ MultiFileInserter.this.notifyAll();
+ }
+ }
+ }
+
+ }
+
+ final FreenetURI targetURI;
+ final HashMap bucketsByName;
+ final InserterContext ctx;
+ final String defaultName;
+ final HashMap namesToMetadataByteArrays;
+ final FailureCodeTracker errorCodes;
+ private int errors;
+ private final boolean[] finished;
+
+ public MultiFileInserter(FreenetURI insertURI, HashMap bucketsByName,
InserterContext context, String defaultName) {
+ this.targetURI = insertURI;
+ this.bucketsByName = bucketsByName;
+ this.ctx = context;
+ this.defaultName = defaultName;
+ this.namesToMetadataByteArrays = new HashMap();
+ this.errorCodes = new FailureCodeTracker(true);
+ if(bucketsByName.get(defaultName) == null)
+ // FIXME make this an InserterException.
+ throw new IllegalArgumentException();
+ finished = new boolean[5];
+ }
+
+ public FreenetURI run() throws InserterException {
+ // For each file, guess MIME type, insert it, get the metadata.
+ // Then put all the metadata at once into one manifest.
+ // Then return it.
+
+ // FIXME scaling issues; have to keep everything in RAM...
+
+ for(int j=0;j<finished.length;j++) {
+ MFInserter it = new MFInserter(j);
+ Thread t = new Thread(it, "Inserter #"+j);
+ t.setDaemon(true);
+ t.start();
+ }
+
+ synchronized(this) {
+ while(true) {
+ boolean stillRunning = false;
+ for(int i=0;i<finished.length;i++) {
+ if(!finished[i]) stillRunning = true;
+ }
+ if(!stillRunning) break;
+ try {
+ wait(10000);
+ } catch (InterruptedException e) {
+ // Impossible??
+ }
+ }
+ }
+
+ if(defaultName != null) {
+ synchronized(namesToMetadataByteArrays) {
+ byte[] defaultData = (byte[])
namesToMetadataByteArrays.get(defaultName);
+ if(defaultData != null)
+ namesToMetadataByteArrays.put("",
defaultData);
+ else {
+ Logger.error(this, "Default name
"+defaultName+" does not exist");
+
if(namesToMetadataByteArrays.containsKey(defaultName))
+ Logger.error(this, "Default
name exists but has null bytes!");
+ // It existed ... and now it doesn't?!
+ throw new
InserterException(InserterException.INTERNAL_ERROR);
+ }
+ }
+ }
+
+ Metadata manifestMetadata =
Metadata.mkRedirectionManifestWithMetadata(namesToMetadataByteArrays);
+
+ Bucket metadata = new
ArrayBucket(manifestMetadata.writeToByteArray());
+
+ FileInserter fi = new FileInserter(ctx);
+
+ InsertBlock block = new InsertBlock(metadata, null, targetURI);
+
+ FreenetURI uri = fi.run(block, true, false, false, null);
+
+ if(errors > 0) {
+ throw new
InserterException(InserterException.FATAL_ERRORS_IN_BLOCKS, errorCodes, uri);
+ }
+
+ return uri;
+ }
+
+ private byte[] getMetadata(String name, Bucket data, String mimeType)
throws InserterException {
+ FileInserter fi = new FileInserter(ctx);
+ InsertBlock block = new InsertBlock(data, new
ClientMetadata(mimeType), FreenetURI.EMPTY_CHK_URI);
+ ArrayBucket metaBucket = new ArrayBucket();
+ FreenetURI uri;
+ // FIXME make a client event and switch this to logger.log(...)
+ System.out.println("Inserting "+name+" ("+data.size()+" bytes,
"+mimeType+")");
+ try {
+ uri = fi.run(block, false, false, false, metaBucket);
+ } catch (InserterException e1) {
+ if(e1.uri != null && e1.getMode() ==
InserterException.COLLISION || e1.getMode() ==
InserterException.ROUTE_NOT_FOUND || e1.getMode() ==
InserterException.ROUTE_REALLY_NOT_FOUND) {
+ Logger.minor(this, "Ignoring "+e1);
+ uri = e1.uri;
+ } else {
+ // Clear the uri.
+ throw new InserterException(e1.getMode());
+ }
+ }
+ byte[] metaByteArray;
+ if(metaBucket.size() == 0) {
+ // It didn't give us any metadata
+ Logger.minor(this, "Did not return metadata: creating
our own");
+ Metadata m = new Metadata(Metadata.SIMPLE_REDIRECT,
uri, null);
+ metaByteArray = m.writeToByteArray();
+ if(metaByteArray == null) throw new
NullPointerException();
+ } else {
+ try {
+ metaByteArray =
BucketTools.toByteArray(metaBucket);
+ if(metaByteArray == null) throw new
NullPointerException();
+ } catch (IOException e) {
+ throw new Error(e);
+ }
+ }
+ return metaByteArray;
+ }
+
+}
Modified: trunk/freenet/src/freenet/client/SplitInserter.java
===================================================================
--- trunk/freenet/src/freenet/client/SplitInserter.java 2006-01-14 17:36:25 UTC
(rev 7856)
+++ trunk/freenet/src/freenet/client/SplitInserter.java 2006-01-14 20:24:24 UTC
(rev 7857)
@@ -1,5 +1,6 @@
package freenet.client;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Vector;
@@ -28,6 +29,7 @@
final int checkSegmentSize;
final int blockSize;
final boolean isMetadata;
+ final Bucket returnMetadata;
SplitfileBlock[] origDataBlocks;
InsertSegment encodingSegment;
InsertSegment[] segments;
@@ -40,7 +42,11 @@
private SplitfileBlock[] fatalErrorBlocks;
private FileInserter inserter;
- public SplitInserter(Bucket data, ClientMetadata clientMetadata,
Compressor compressor, short splitfileAlgorithm, InserterContext ctx,
FileInserter inserter, int blockLength, boolean getCHKOnly, boolean isMetadata)
throws InserterException {
+ /**
+ * @param returnMetadata If not null, then write the metadata to this
bucket,
+ * rather than inserting it.
+ */
+ public SplitInserter(Bucket data, ClientMetadata clientMetadata,
Compressor compressor, short splitfileAlgorithm, InserterContext ctx,
FileInserter inserter, int blockLength, boolean getCHKOnly, boolean isMetadata,
Bucket returnMetadata) throws InserterException {
this.origData = data;
this.getCHKOnly = getCHKOnly;
this.blockSize = blockLength;
@@ -62,6 +68,7 @@
}
this.inserter = inserter;
this.isMetadata = isMetadata;
+ this.returnMetadata = returnMetadata;
}
/**
@@ -122,31 +129,45 @@
Metadata metadata = new Metadata(splitfileAlgorithm,
dataURIs, checkURIs, segmentSize, checkSegmentSize, clientMetadata, dataLength,
compressionCodec, isMetadata);
- Bucket mbucket;
- try {
- mbucket =
BucketTools.makeImmutableBucket(ctx.bf, metadata.writeToByteArray());
- } catch (IOException e) {
- throw new
InserterException(InserterException.BUCKET_ERROR, null);
- }
+ if(returnMetadata != null) {
+ DataOutputStream dos;
+ try {
+ dos = new
DataOutputStream(returnMetadata.getOutputStream());
+ metadata.writeTo(dos);
+ dos.close();
+ } catch (IOException e) {
+ throw new
InserterException(InserterException.BUCKET_ERROR);
+ }
+ } else {
- if(inserter == null)
- inserter = new FileInserter(ctx);
-
- InsertBlock mblock = new InsertBlock(mbucket, null,
FreenetURI.EMPTY_CHK_URI);
-
- // FIXME probably should uncomment below so it doesn't
get inserted at all?
- // FIXME this is a hack for small network support...
but we will need that IRL... hmmm
- try {
- uri = inserter.run(mblock, true, getCHKOnly/*
|| (fatalErrors > 0 || failed > 0)*/, false);
- } catch (InserterException e) {
- e.errorCodes =
tracker.getAccumulatedNonFatalErrorCodes().merge(tracker.getAccumulatedFatalErrorCodes());
- throw e;
+ Bucket mbucket;
+ try {
+ mbucket =
BucketTools.makeImmutableBucket(ctx.bf, metadata.writeToByteArray());
+ } catch (IOException e) {
+ throw new
InserterException(InserterException.BUCKET_ERROR, null);
+ }
+
+ if(inserter == null)
+ inserter = new FileInserter(ctx);
+
+ InsertBlock mblock = new InsertBlock(mbucket,
null, FreenetURI.EMPTY_CHK_URI);
+
+ // FIXME probably should uncomment below so it
doesn't get inserted at all?
+ // FIXME this is a hack for small network
support... but we will need that IRL... hmmm
+ try {
+ uri = inserter.run(mblock, true,
getCHKOnly/* || (fatalErrors > 0 || failed > 0)*/, false, null);
+ } catch (InserterException e) {
+ e.errorCodes =
tracker.getAccumulatedNonFatalErrorCodes().merge(tracker.getAccumulatedFatalErrorCodes());
+ throw e;
+ }
}
+
}
// Did we succeed?
- ctx.eventProducer.produceEvent(new GeneratedURIEvent(uri));
+ if(uri != null)
+ ctx.eventProducer.produceEvent(new
GeneratedURIEvent(uri));
if(fatalErrors > 0) {
throw new
InserterException(InserterException.FATAL_ERRORS_IN_BLOCKS,
tracker.getAccumulatedFatalErrorCodes(), uri);
Modified: trunk/freenet/src/freenet/clients/http/ToadletContextImpl.java
===================================================================
--- trunk/freenet/src/freenet/clients/http/ToadletContextImpl.java
2006-01-14 17:36:25 UTC (rev 7856)
+++ trunk/freenet/src/freenet/clients/http/ToadletContextImpl.java
2006-01-14 20:24:24 UTC (rev 7857)
@@ -12,6 +12,8 @@
import freenet.support.BucketTools;
import freenet.support.Logger;
import freenet.support.MultiValueTable;
+import freenet.support.URLDecoder;
+import freenet.support.URLEncodedFormatException;
import freenet.support.io.LineReadingInputStream;
import freenet.support.io.TooLongException;
@@ -131,10 +133,13 @@
URI uri;
try {
- uri = new URI(split[1]);
+ uri = new
URI(URLDecoder.decode(split[1]));
} catch (URISyntaxException e) {
sendURIParseError(sock.getOutputStream(), true);
return;
+ } catch (URLEncodedFormatException e) {
+
sendURIParseError(sock.getOutputStream(), true);
+ return;
}
String method = split[0];
Modified: trunk/freenet/src/freenet/node/TextModeClientInterface.java
===================================================================
--- trunk/freenet/src/freenet/node/TextModeClientInterface.java 2006-01-14
17:36:25 UTC (rev 7856)
+++ trunk/freenet/src/freenet/node/TextModeClientInterface.java 2006-01-14
20:24:24 UTC (rev 7857)
@@ -284,61 +284,41 @@
line = line.split("#")[0];
}
- // Get files as name and keys
- HashMap manifestBase = dirPut(line, getCHKOnly);
+ HashMap bucketsByName =
+ makeBucketsByName(line);
- // Set defaultfile
- if (defaultFile != null) {
- HashMap currPos = manifestBase;
- String splitpath[] = defaultFile.split("/");
- int i = 0;
- for( ; i < (splitpath.length - 1) ; i++)
- currPos = (HashMap)currPos.get(splitpath[i]);
-
- if (currPos.get(splitpath[i]) != null) {
- // Add key as default
- manifestBase.put("", currPos.get(splitpath[i]));
- System.out.println("Using default key: " +
currPos.get(splitpath[i]));
- }else{
- System.err.println("Default key not found. No
default document.");
+ if(defaultFile == null) {
+ String[] defaultFiles =
+ new String[] { "index.html", "index.htm",
"default.html", "default.htm" };
+ for(int i=0;i<defaultFiles.length;i++) {
+ if(bucketsByName.containsKey(defaultFiles[i])) {
+ defaultFile = defaultFiles[i];
+ break;
+ }
}
- //getchkdir:/home/cyberdo/fort/new#filelist
}
- // Create metadata
- Metadata med = Metadata.mkRedirectionManifest(manifestBase);
- ClientMetadata md = med.getClientMetadata();
-
- // Extract binary data from metadata
- ArrayBucket metabucket = new ArrayBucket();
- DataOutputStream mdos = new DataOutputStream(
metabucket.getOutputStream() );
- med.writeTo(mdos);
- mdos.close();
-
- // Insert metadata
- InsertBlock block = new InsertBlock(metabucket, md,
FreenetURI.EMPTY_CHK_URI);
-
- FreenetURI uri;
- try {
- uri = ((HighLevelSimpleClientImpl)client).insert(block,
getCHKOnly, true);
- } catch (InserterException e) {
- System.out.println("Error: "+e.getMessage());
- if(e.uri != null)
+ FreenetURI uri;
+ try {
+ uri =
client.insertManifest(FreenetURI.EMPTY_CHK_URI, bucketsByName, defaultFile);
+ uri.addMetaStrings(new String[] { "" });
+
System.out.println("=======================================================");
+ System.out.println("URI: "+uri);
+
System.out.println("=======================================================");
+ } catch (InserterException e) {
+ System.out.println("Finished insert but: "+e.getMessage());
+ if(e.uri != null) {
+ uri = e.uri;
+ uri.addMetaStrings(new String[] { "" });
System.out.println("URI would have been: "+e.uri);
- int mode = e.getMode();
- if(mode == InserterException.FATAL_ERRORS_IN_BLOCKS || mode ==
InserterException.TOO_MANY_RETRIES_IN_BLOCKS) {
- System.out.println("Splitfile-specific
error:\n"+e.errorCodes.toVerboseString());
}
- return;
- }
+ if(e.errorCodes != null) {
+ System.out.println("Splitfile errors breakdown:");
+ System.out.println(e.errorCodes.toVerboseString());
+ }
+ Logger.error(this, "Caught "+e, e);
+ }
- String filelist = dirPutToList(manifestBase, "");
-
System.out.println("=======================================================");
- System.out.println(filelist);
-
System.out.println("=======================================================");
- System.out.println("URI: "+uri);
-
System.out.println("=======================================================");
-
} else if(uline.startsWith("PUTFILE:") || (getCHKOnly =
uline.startsWith("GETCHKFILE:"))) {
// Just insert to local store
if(getCHKOnly) {
@@ -481,8 +461,42 @@
}
}
-
- private String dirPutToList(HashMap dir, String basedir) {
+ /**
+ * Create a map of String -> Bucket for every file in a directory
+ * and its subdirs.
+ */
+ private HashMap makeBucketsByName(String directory) {
+
+ if (!directory.endsWith("/"))
+ directory = directory + "/";
+ File thisdir = new File(directory);
+
+ System.out.println("Listing dir: "+thisdir);
+
+ HashMap ret = new HashMap();
+
+ File filelist[] = thisdir.listFiles();
+ for(int i = 0 ; i < filelist.length ; i++) {
+ if (filelist[i].isFile() && filelist[i].canRead()) {
+ File f = filelist[i];
+
+ FileBucket bucket = new FileBucket(f, true, false,
false);
+
+ ret.put(f.getName(), bucket);
+ } else if(filelist[i].isDirectory()) {
+ HashMap subdir = makeBucketsByName(directory +
filelist[i].getName());
+ Iterator it = subdir.keySet().iterator();
+ while(it.hasNext()) {
+ String key = (String) it.next();
+ Bucket bucket = (Bucket) subdir.get(key);
+ ret.put(filelist[i].getName() + "/" + key,
bucket);
+ }
+ }
+ }
+ return ret;
+ }
+
+ private String dirPutToList(HashMap dir, String basedir) {
String ret = "";
for(Iterator i = dir.keySet().iterator();i.hasNext();) {
String key = (String) i.next();
Modified: trunk/freenet/src/freenet/node/Version.java
===================================================================
--- trunk/freenet/src/freenet/node/Version.java 2006-01-14 17:36:25 UTC (rev
7856)
+++ trunk/freenet/src/freenet/node/Version.java 2006-01-14 20:24:24 UTC (rev
7857)
@@ -20,7 +20,7 @@
public static final String protocolVersion = "1.0";
/** The build number of the current revision */
- public static final int buildNumber = 352;
+ public static final int buildNumber = 353;
/** Oldest build of Fred we will talk to */
public static final int lastGoodBuild = 348;
