Author: orbiter
Date: 2008-03-14 01:16:04 +0100 (Fri, 14 Mar 2008)
New Revision: 4558
Modified:
trunk/htroot/WatchCrawler_p.java
trunk/htroot/xml/util/getpageinfo_p.java
trunk/htroot/yacy/transfer.java
trunk/source/de/anomic/http/httpTemplate.java
trunk/source/de/anomic/http/httpd.java
trunk/source/de/anomic/http/httpdFileHandler.java
trunk/source/de/anomic/index/indexRAMRI.java
trunk/source/de/anomic/kelondro/kelondroAttrSeq.java
trunk/source/de/anomic/kelondro/kelondroRowCollection.java
trunk/source/de/anomic/plasma/parser/mimeType/odtDetector.java
trunk/source/de/anomic/plasma/plasmaHTCache.java
trunk/source/de/anomic/server/serverCore.java
trunk/source/de/anomic/server/serverFileUtils.java
trunk/source/de/anomic/server/serverSystem.java
trunk/source/yacy.java
Log:
- next try to fix the networking problem:
set the maximum transfer size to less than MTU=1500-52: buffer size <= 1448
- some refactoring of transfer methods (naming)
Modified: trunk/htroot/WatchCrawler_p.java
===================================================================
--- trunk/htroot/WatchCrawler_p.java 2008-03-12 21:24:53 UTC (rev 4557)
+++ trunk/htroot/WatchCrawler_p.java 2008-03-14 00:16:04 UTC (rev 4558)
@@ -262,7 +262,7 @@
htmlFilterContentScraper scraper = new
htmlFilterContentScraper(new yacyURL(file));
//OutputStream os = new
htmlFilterOutputStream(null, scraper, null, false);
Writer writer = new
htmlFilterWriter(null,null,scraper,null,false);
- serverFileUtils.write(fileString,writer);
+ serverFileUtils.copy(fileString, writer);
writer.close();
//String headline = scraper.getHeadline();
Modified: trunk/htroot/xml/util/getpageinfo_p.java
===================================================================
--- trunk/htroot/xml/util/getpageinfo_p.java 2008-03-12 21:24:53 UTC (rev
4557)
+++ trunk/htroot/xml/util/getpageinfo_p.java 2008-03-14 00:16:04 UTC (rev
4558)
@@ -88,7 +88,7 @@
htmlFilterContentScraper scraper = new
htmlFilterContentScraper(u);
//OutputStream os = new htmlFilterOutputStream(null,
scraper, null, false);
Writer writer = new
htmlFilterWriter(null,null,scraper,null,false);
- serverFileUtils.write(contentString,writer);
+ serverFileUtils.copy(contentString,writer);
writer.close();
// put the document title
Modified: trunk/htroot/yacy/transfer.java
===================================================================
--- trunk/htroot/yacy/transfer.java 2008-03-12 21:24:53 UTC (rev 4557)
+++ trunk/htroot/yacy/transfer.java 2008-03-14 00:16:04 UTC (rev 4558)
@@ -137,7 +137,7 @@
File file = new File(path, filename);
try {
if
(file.getCanonicalPath().toString().startsWith(path.getCanonicalPath().toString())){
- serverFileUtils.write(fileString.getBytes(), file);
+ serverFileUtils.copy(fileString.getBytes(), file);
String md5t = serverCodings.encodeMD5Hex(file);
if (md5t.equals(md5)) {
prop.put("response", "ok");
Modified: trunk/source/de/anomic/http/httpTemplate.java
===================================================================
--- trunk/source/de/anomic/http/httpTemplate.java 2008-03-12 21:24:53 UTC
(rev 4557)
+++ trunk/source/de/anomic/http/httpTemplate.java 2008-03-14 00:16:04 UTC
(rev 4558)
@@ -435,7 +435,7 @@
}
*/
- serverFileUtils.write(replacement, out);
+ serverFileUtils.copy(replacement, out);
} else {
// inconsistency, simply finalize this
serverFileUtils.copy(pis, out);
@@ -479,7 +479,7 @@
byte[] tmp=new byte[2];
tmp[0]=hash;
tmp[1]=(byte)bb;
- serverFileUtils.write(tmp, out);
+ serverFileUtils.copy(tmp, out);
}
}
//System.out.println(structure.toString()); //DEBUG
Modified: trunk/source/de/anomic/http/httpd.java
===================================================================
--- trunk/source/de/anomic/http/httpd.java 2008-03-12 21:24:53 UTC (rev
4557)
+++ trunk/source/de/anomic/http/httpd.java 2008-03-14 00:16:04 UTC (rev
4558)
@@ -1236,7 +1236,7 @@
if (! method.equals(httpHeader.METHOD_HEAD)) {
// write the array to the client
- serverFileUtils.write(result, respond);
+ serverFileUtils.copy(result, respond);
}
respond.flush();
} catch (Exception e) {
Modified: trunk/source/de/anomic/http/httpdFileHandler.java
===================================================================
--- trunk/source/de/anomic/http/httpdFileHandler.java 2008-03-12 21:24:53 UTC
(rev 4557)
+++ trunk/source/de/anomic/http/httpdFileHandler.java 2008-03-14 00:16:04 UTC
(rev 4558)
@@ -768,7 +768,7 @@
httpVersion, 200, null, mimeType,
result.length,
targetDate, null, tp.getOutgoingHeader(),
contentEncoding, null, nocache);
- serverFileUtils.write(result, out);
+ serverFileUtils.copy(result, out);
}
}
} else { // no html
Modified: trunk/source/de/anomic/index/indexRAMRI.java
===================================================================
--- trunk/source/de/anomic/index/indexRAMRI.java 2008-03-12 21:24:53 UTC
(rev 4557)
+++ trunk/source/de/anomic/index/indexRAMRI.java 2008-03-14 00:16:04 UTC
(rev 4558)
@@ -178,7 +178,7 @@
if (writeBuffer != null) {
serverByteBuffer bb = writeBuffer.getBuffer();
//System.out.println("*** byteBuffer size = " + bb.length());
- serverFileUtils.write(bb.getBytes(), indexDumpFile);
+ serverFileUtils.copy(bb.getBytes(), indexDumpFile);
writeBuffer.close();
}
dumpArray.close();
Modified: trunk/source/de/anomic/kelondro/kelondroAttrSeq.java
===================================================================
--- trunk/source/de/anomic/kelondro/kelondroAttrSeq.java 2008-03-12
21:24:53 UTC (rev 4557)
+++ trunk/source/de/anomic/kelondro/kelondroAttrSeq.java 2008-03-14
00:16:04 UTC (rev 4558)
@@ -204,7 +204,7 @@
if (out.toString().endsWith(".gz")) {
serverFileUtils.writeAndGZip((new String(sb)).getBytes(), out);
} else {
- serverFileUtils.write((new String(sb)).getBytes(), out);
+ serverFileUtils.copy((new String(sb)).getBytes(), out);
}
}
Modified: trunk/source/de/anomic/kelondro/kelondroRowCollection.java
===================================================================
--- trunk/source/de/anomic/kelondro/kelondroRowCollection.java 2008-03-12
21:24:53 UTC (rev 4557)
+++ trunk/source/de/anomic/kelondro/kelondroRowCollection.java 2008-03-14
00:16:04 UTC (rev 4558)
@@ -183,7 +183,7 @@
}
public void saveCollection(File file) throws IOException {
- serverFileUtils.write(exportCollection(), file);
+ serverFileUtils.copy(exportCollection(), file);
}
public kelondroRow row() {
Modified: trunk/source/de/anomic/plasma/parser/mimeType/odtDetector.java
===================================================================
--- trunk/source/de/anomic/plasma/parser/mimeType/odtDetector.java
2008-03-12 21:24:53 UTC (rev 4557)
+++ trunk/source/de/anomic/plasma/parser/mimeType/odtDetector.java
2008-03-14 00:16:04 UTC (rev 4558)
@@ -81,7 +81,7 @@
File dstFile = null;
try {
dstFile = File.createTempFile("mimeTypeParser",".tmp");
- serverFileUtils.write(data,dstFile);
+ serverFileUtils.copy(data,dstFile);
return process(dstFile, offset, length, bitmask, comparator,
mimeType, params);
} catch (IOException e) {
return null;
Modified: trunk/source/de/anomic/plasma/plasmaHTCache.java
===================================================================
--- trunk/source/de/anomic/plasma/plasmaHTCache.java 2008-03-12 21:24:53 UTC
(rev 4557)
+++ trunk/source/de/anomic/plasma/plasmaHTCache.java 2008-03-14 00:16:04 UTC
(rev 4558)
@@ -373,7 +373,7 @@
try {
deleteFile(file);
file.getParentFile().mkdirs();
- serverFileUtils.write(array, file);
+ serverFileUtils.copy(array, file);
} catch (FileNotFoundException e) {
// this is the case of a "(Not a directory)" error, which should
be prohibited
// by the shallStoreCache() property. However, sometimes the error
still occurs
Modified: trunk/source/de/anomic/server/serverCore.java
===================================================================
--- trunk/source/de/anomic/server/serverCore.java 2008-03-12 21:24:53 UTC
(rev 4557)
+++ trunk/source/de/anomic/server/serverCore.java 2008-03-14 00:16:04 UTC
(rev 4558)
@@ -422,6 +422,9 @@
// set a non-zero linger, that means that a socket.close()
blocks until all data is written
controlSocket.setSoLinger(true, this.timeout);
+ // ensure that MTU-48 is not exceeded to prevent that routers
cannot handle large data packets
+ controlSocket.setSendBufferSize(1440);
+
// create session
Session connection = new Session(sessionThreadGroup,
controlSocket, this.timeout);
this.busySessions.add(connection);
Modified: trunk/source/de/anomic/server/serverFileUtils.java
===================================================================
--- trunk/source/de/anomic/server/serverFileUtils.java 2008-03-12 21:24:53 UTC
(rev 4557)
+++ trunk/source/de/anomic/server/serverFileUtils.java 2008-03-14 00:16:04 UTC
(rev 4558)
@@ -72,10 +72,10 @@
public final class serverFileUtils {
- private static final int DEFAULT_BUFFER_SIZE = 512;
+ private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the
maximum chunk size
public static long copy(InputStream source, OutputStream dest) throws
IOException {
- return copy(source,dest, -1);
+ return copy(source, dest, -1);
}
/**
@@ -133,7 +133,7 @@
}
public static int copy(Reader source, Writer dest) throws IOException {
- char[] buffer = new char[4096];
+ char[] buffer = new char[DEFAULT_BUFFER_SIZE];
int count = 0;
int n = 0;
try {
@@ -151,7 +151,7 @@
public static void copy(InputStream source, File dest) throws IOException {
copy(source,dest,-1);
}
-
+
/**
* Copies an InputStream to a File.
* @param source InputStream
@@ -213,10 +213,6 @@
}
}
- public static void copy(File source, File dest) throws IOException {
- copy(source,dest,-1);
- }
-
/**
* Copies a File to a File.
* @param source File
@@ -227,19 +223,28 @@
* @see #copyRange(File source, OutputStream dest, int start)
* @see #copy(File source, OutputStream dest)
*/
- public static void copy(File source, File dest, long count) throws
IOException {
+ public static void copy(File source, File dest) throws IOException {
FileInputStream fis = null;
FileOutputStream fos = null;
try {
fis = new FileInputStream(source);
fos = new FileOutputStream(dest);
- copy(fis, fos, count);
+ copy(fis, fos, -1);
} finally {
if (fis != null) try {fis.close();} catch (Exception e) {}
if (fos != null) try {fos.close();} catch (Exception e) {}
}
}
+ public static void copy(byte[] source, OutputStream dest) throws
IOException {
+ dest.write(source, 0, source.length);
+ dest.flush();
+ }
+
+ public static void copy(byte[] source, File dest) throws IOException {
+ copy(new ByteArrayInputStream(source), dest);
+ }
+
public static byte[] read(InputStream source) throws IOException {
return read(source,-1);
}
@@ -297,27 +302,13 @@
GZIPOutputStream zipOut = null;
try {
zipOut = new GZIPOutputStream(dest);
- write(source, zipOut);
+ copy(source, zipOut);
zipOut.close();
} finally {
if (zipOut != null) try { zipOut.close(); } catch (Exception e) {}
}
}
- public static void write(String source, Writer dest) throws IOException {
- dest.write(source);
- dest.flush();
- }
-
- public static void write(byte[] source, OutputStream dest) throws
IOException {
- dest.write(source, 0, source.length);
- dest.flush();
- }
-
- public static void write(byte[] source, File dest) throws IOException {
- copy(new ByteArrayInputStream(source), dest);
- }
-
/**
* This function determines if a byte array is gzip compressed and
uncompress it
* @param source properly gzip compressed byte array
Modified: trunk/source/de/anomic/server/serverSystem.java
===================================================================
--- trunk/source/de/anomic/server/serverSystem.java 2008-03-12 21:24:53 UTC
(rev 4557)
+++ trunk/source/de/anomic/server/serverSystem.java 2008-03-14 00:16:04 UTC
(rev 4558)
@@ -328,7 +328,7 @@
}
public static void deployScript(File scriptFile, String theScript) throws
IOException {
- serverFileUtils.write(theScript.getBytes(), scriptFile);
+ serverFileUtils.copy(theScript.getBytes(), scriptFile);
try {
Runtime.getRuntime().exec("chmod 755 " +
scriptFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor();
} catch (InterruptedException e) {
Modified: trunk/source/yacy.java
===================================================================
--- trunk/source/yacy.java 2008-03-12 21:24:53 UTC (rev 4557)
+++ trunk/source/yacy.java 2008-03-14 00:16:04 UTC (rev 4558)
@@ -286,7 +286,7 @@
} catch (IOException e) {}
final File htdocsReadme = new File(htDocsPath, "readme.txt");
- if (!(htdocsReadme.exists())) try {serverFileUtils.write((
+ if (!(htdocsReadme.exists())) try {serverFileUtils.copy((
"This is your root directory for individual Web
Content\r\n" +
"\r\n" +
"Please place your html files into the www
subdirectory.\r\n" +
_______________________________________________
YaCy-svn mailing list
[email protected]
https://lists.berlios.de/mailman/listinfo/yacy-svn
