Author: orbiter
Date: 2008-02-28 00:12:43 +0100 (Thu, 28 Feb 2008)
New Revision: 4519
Modified:
trunk/source/de/anomic/http/httpc.java
trunk/source/de/anomic/http/httpdByteCountInputStream.java
trunk/source/de/anomic/kelondro/kelondroSortStore.java
trunk/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java
trunk/source/de/anomic/server/serverCore.java
trunk/source/de/anomic/server/serverFileUtils.java
trunk/source/de/anomic/tools/crypt.java
trunk/source/de/anomic/tools/cryptbig.java
trunk/source/de/anomic/tools/gzip.java
trunk/source/de/anomic/yacy/yacyClient.java
Log:
- fixed a bug in remote search that prevented that any results had been
generated (!)
- added a great number of printStackTrace and new exceptions that shall be used
to find the cause
for a bug in yacy client-server communication which causes the interruption
of data transfer
which then causes the parser bug for the seed strings.
- tried to fix the communication bug on server-side (copy functions)
Be aware that the log may be full of errors and bugs - there should not be more
bugs but there is more to see
Modified: trunk/source/de/anomic/http/httpc.java
===================================================================
--- trunk/source/de/anomic/http/httpc.java 2008-02-27 20:09:22 UTC (rev
4518)
+++ trunk/source/de/anomic/http/httpc.java 2008-02-27 23:12:43 UTC (rev
4519)
@@ -400,6 +400,7 @@
this.socket.setSoTimeout(timeout); // waiting time for read
// get the connection
this.socket.connect(address, timeout);
+ this.socket.setSoTimeout(timeout); // waiting time for read
if (incomingByteCountAccounting != null) {
this.clientInputByteCount = new
httpdByteCountInputStream(this.socket.getInputStream(),incomingByteCountAccounting);
@@ -411,7 +412,7 @@
// getting input and output streams
this.clientInput = new
PushbackInputStream((this.clientInputByteCount!=null)?
this.clientInputByteCount:
- this.socket.getInputStream());
+ this.socket.getInputStream());
this.clientOutput = this.socket.getOutputStream();
// if we reached this point, we should have a connection
@@ -422,7 +423,7 @@
} catch (IOException e) {
// There was an error while connecting the socket, probably a
SocketTimeoutException
// we have to close the httpc, otherwise it would stay in
activeConnections forever
- serverLog.logFine("HTTPC", "Couldn't open socket to: " +
this.adressed_host + ":" + this.adressed_port);
+ serverLog.logFine("HTTPC", "Couldn't open socket to " +
this.adressed_host + ":" + this.adressed_port + ": " + e.getMessage());
close();
// TODO do we need to hand it over to the caller?
@@ -979,6 +980,8 @@
// read connection body and return body
serverByteBuffer sbb = new serverByteBuffer();
res.writeContent(sbb, null);
+ if ((res.responseHeader.contentLength() > 0) &&
(res.responseHeader.contentLength() != sbb.length()))
+ throw new IOException("content length and loaded resource length
from http://"; + realhost + ":" + port + path + " does not match.
HEADER.Content-Length = " + res.responseHeader.contentLength() + ",
resource.length =" + sbb.length());
con.close();
return sbb.getBytes();
}
@@ -1244,7 +1247,7 @@
}
// reads in the http header, right now, right here
- byte[] b = serverCore.receive(httpc.this.clientInput,
terminalMaxLength, false);
+ byte[] b = serverCore.receive(httpc.this.clientInput,
terminalMaxLength, true);
if (b == null) {
// the server has meanwhile disconnected
this.statusCode = 503;
@@ -1263,7 +1266,7 @@
if ((this.statusCode==500)&&(this.statusText.equals("status line
parse error"))) {
// flush in anything that comes without parsing
- while ((b != null) && (b.length != 0)) b =
serverCore.receive(httpc.this.clientInput, terminalMaxLength, false);
+ while ((b != null) && (b.length != 0)) b =
serverCore.receive(httpc.this.clientInput, terminalMaxLength, true);
return; // in bad mood
}
@@ -1271,13 +1274,13 @@
if (this.statusCode == 400) {
// bad request
// flush in anything that comes without parsing
- while ((b = serverCore.receive(httpc.this.clientInput,
terminalMaxLength, false)).length != 0) {}
+ while ((b = serverCore.receive(httpc.this.clientInput,
terminalMaxLength, true)).length != 0) {}
return; // in bad mood
}
// at this point we should have a valid response. read in the
header properties
String key = "";
- while ((b = serverCore.receive(httpc.this.clientInput,
terminalMaxLength, false)) != null) {
+ while ((b = serverCore.receive(httpc.this.clientInput,
terminalMaxLength, true)) != null) {
if (b.length == 0) break;
buffer = new String(b);
buffer=buffer.trim();
@@ -1362,7 +1365,7 @@
return new GZIPInputStream(httpc.this.clientInput);
} else if (this.responseHeader.contentLength() != -1) {
// use a httpContentLengthInputStream to read until the end of
the response body is reached
- return new
httpContentLengthInputStream(httpc.this.clientInput,this.responseHeader.contentLength());
+ return new
httpContentLengthInputStream(httpc.this.clientInput,
this.responseHeader.contentLength());
}
// no Content-Lenght was set. In this case we can read until EOF
return httpc.this.clientInput;
@@ -1429,13 +1432,17 @@
} else {
throw new IllegalArgumentException("Invalid procOS object
type '" + procOS.getClass().getName() + "'");
}
- } catch (IOException e) {}
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
if (bufferOS != null) {
try {
bufferOS.flush();
bufferOS.close();
- } catch (IOException e) {}
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
if (file.length() == 0) file.delete();
}
}
@@ -1453,13 +1460,16 @@
if (System.currentTimeMillis() - lastIO > 30000) break;
this.wait(300);
continue io;
- } catch (InterruptedException e) {} // may happen without EOF
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } // may happen without EOF
lastIO = System.currentTimeMillis();
c += l;
if (procOS != null) procOS.write(buffer, 0, l);
if (bufferOS != null) bufferOS.write(buffer, 0, l);
} catch (IOException e) {
- //System.out.println("*** DEBUG: writeX/IOStream terminated
with IOException, processed " + c + " bytes.");
+ System.out.println("*** DEBUG: writeX/IOStream terminated with
IOException, processed " + c + " bytes. cause: " + e.getMessage());
+ e.printStackTrace();
break;
}
@@ -1484,13 +1494,15 @@
if (System.currentTimeMillis() - lastIO > 30000) break;
this.wait(300);
continue io;
- } catch (InterruptedException e) {} // may happen without
EOF
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } // may happen without EOF
lastIO = System.currentTimeMillis();
c += l;
if (procOS != null) procOS.write(buffer, 0, l);
if (bufferOSWriter != null) bufferOSWriter.write(buffer,
0, l);
} catch (IOException e) {
- //System.out.println("*** DEBUG: writeX/ReaderWriter
terminated with IOException, processed " + c + " bytes.");
+ System.out.println("*** DEBUG: writeX/ReaderWriter
terminated with IOException, processed " + c + " bytes. cause: " +
e.getMessage());
break;
}
Modified: trunk/source/de/anomic/http/httpdByteCountInputStream.java
===================================================================
--- trunk/source/de/anomic/http/httpdByteCountInputStream.java 2008-02-27
20:09:22 UTC (rev 4518)
+++ trunk/source/de/anomic/http/httpdByteCountInputStream.java 2008-02-27
23:12:43 UTC (rev 4519)
@@ -91,9 +91,14 @@
}
public int read(byte[] b, int off, int len) throws IOException {
+ try {
int readCount = super.read(b, off, len);
if (readCount > 0) this.byteCount += readCount;
return readCount;
+ } catch (IOException e) {
+ e.printStackTrace();
+ throw new IOException(e.getMessage() + "; b.length = " + b.length
+ ", off = " + off + ", len = " + len);
+ }
}
public int read() throws IOException {
Modified: trunk/source/de/anomic/kelondro/kelondroSortStore.java
===================================================================
--- trunk/source/de/anomic/kelondro/kelondroSortStore.java 2008-02-27
20:09:22 UTC (rev 4518)
+++ trunk/source/de/anomic/kelondro/kelondroSortStore.java 2008-02-27
23:12:43 UTC (rev 4519)
@@ -100,7 +100,7 @@
return this.offstack;
}
if (size() < count) throw new RuntimeException("list(" + count + ")
exceeded avaiable number of elements (" + size() + ")");
- while (this.onstack.size() < count) {
+ while (this.offstack.size() < count) {
Long w = this.onstack.firstKey();
E element = this.onstack.remove(w);
stackElement se = new stackElement(element, w);
Modified: trunk/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java
===================================================================
--- trunk/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java
2008-02-27 20:09:22 UTC (rev 4518)
+++ trunk/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java
2008-02-27 23:12:43 UTC (rev 4519)
@@ -178,8 +178,6 @@
log.logFine("CoreCrawl: online caution, omitting processing");
return false;
}
- // if the server is busy, we do crawling more slowly
- //if (!(cacheManager.idle())) try
{Thread.currentThread().sleep(2000);} catch (InterruptedException e) {}
// if crawling was paused we have to wait until we wer notified to
continue
Object[] status = (Object[])
sb.crawlJobsStatus.get(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
Modified: trunk/source/de/anomic/server/serverCore.java
===================================================================
--- trunk/source/de/anomic/server/serverCore.java 2008-02-27 20:09:22 UTC
(rev 4518)
+++ trunk/source/de/anomic/server/serverCore.java 2008-02-27 23:12:43 UTC
(rev 4519)
@@ -601,7 +601,7 @@
}
public byte[] readLine() {
- return receive(this.in, serverCore.this.commandMaxLength, false);
+ return receive(this.in, serverCore.this.commandMaxLength, true);
}
/**
Modified: trunk/source/de/anomic/server/serverFileUtils.java
===================================================================
--- trunk/source/de/anomic/server/serverFileUtils.java 2008-02-27 20:09:22 UTC
(rev 4518)
+++ trunk/source/de/anomic/server/serverFileUtils.java 2008-02-27 23:12:43 UTC
(rev 4519)
@@ -72,7 +72,7 @@
public final class serverFileUtils {
- private static final int DEFAULT_BUFFER_SIZE = 4096;
+ private static final int DEFAULT_BUFFER_SIZE = 512;
public static long copy(InputStream source, OutputStream dest) throws
IOException {
return copy(source,dest, -1);
@@ -95,13 +95,13 @@
int chunkSize = (int) ((count > 0) ? Math.min(count,
DEFAULT_BUFFER_SIZE) : DEFAULT_BUFFER_SIZE);
int c; long total = 0;
- while ((c = source.read(buffer,0,chunkSize)) > 0) {
+ while ((c = source.read(buffer, 0, chunkSize)) > 0) {
dest.write(buffer, 0, c);
dest.flush();
total += c;
if (count > 0) {
- chunkSize = (int)Math.min(count-total,DEFAULT_BUFFER_SIZE);
+ chunkSize = (int) Math.min(count-total, DEFAULT_BUFFER_SIZE);
if (chunkSize == 0) break;
}
@@ -111,7 +111,7 @@
return total;
}
- public static int copy (File source, String inputCharset, Writer dest)
throws IOException {
+ public static int copy(File source, String inputCharset, Writer dest)
throws IOException {
InputStream fis = null;
try {
fis = new FileInputStream(source);
@@ -121,18 +121,18 @@
}
}
- public static int copy (InputStream source, Writer dest, String
inputCharset) throws IOException {
+ public static int copy(InputStream source, Writer dest, String
inputCharset) throws IOException {
InputStreamReader reader = new InputStreamReader(source,inputCharset);
return copy(reader,dest);
}
- public static int copy (String source, Writer dest) throws IOException {
+ public static int copy(String source, Writer dest) throws IOException {
dest.write(source);
dest.flush();
return source.length();
}
- public static int copy (Reader source, Writer dest) throws IOException {
+ public static int copy(Reader source, Writer dest) throws IOException {
char[] buffer = new char[4096];
int count = 0;
int n = 0;
@@ -188,7 +188,7 @@
fis = new FileInputStream(source);
long skipped = fis.skip(start);
if (skipped != start) throw new IllegalStateException("Unable to
skip '" + start + "' bytes. Only '" + skipped + "' bytes skipped.");
- copy(fis, dest,-1);
+ copy(fis, dest, -1);
} finally {
if (fis != null) try { fis.close(); } catch (Exception e) {}
}
@@ -305,11 +305,13 @@
}
public static void write(String source, Writer dest) throws IOException {
- copy(source,dest);
+ dest.write(source);
+ dest.flush();
}
public static void write(byte[] source, OutputStream dest) throws
IOException {
- copy(new ByteArrayInputStream(source), dest, -1);
+ dest.write(source, 0, source.length);
+ dest.flush();
}
public static void write(byte[] source, File dest) throws IOException {
@@ -327,6 +329,7 @@
// support of gzipped data (requested by roland)
if ((source.length > 1) && (((source[1] << 8) | source[0]) ==
GZIPInputStream.GZIP_MAGIC)) {
+ System.out.println("DEBUG: uncompressGZipArray - uncompressing
source");
try {
ByteArrayInputStream byteInput = new
ByteArrayInputStream(source);
ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
Modified: trunk/source/de/anomic/tools/crypt.java
===================================================================
--- trunk/source/de/anomic/tools/crypt.java 2008-02-27 20:09:22 UTC (rev
4518)
+++ trunk/source/de/anomic/tools/crypt.java 2008-02-27 23:12:43 UTC (rev
4519)
@@ -119,15 +119,31 @@
}
public static String simpleDecode(String encoded, String key) {
- if (encoded == null || encoded.length() < 3) { return null; }
- if (encoded.charAt(1) != '|') { return encoded; } // not encoded
- switch (encoded.charAt(0)) {
- case 'b' : return
kelondroBase64Order.enhancedCoder.decodeString(encoded.substring(2),
"de.anomic.tools.crypt.simpleDecode()");
- case 'z' : return
gzip.gunzipString(kelondroBase64Order.enhancedCoder.decode(encoded.substring(2),
"de.anomic.tools.crypt.simpleDecode()"));
- case 'p' : return encoded.substring(2);
- default : return null;
+ if (encoded == null || encoded.length() < 3) {
+ return null;
+ }
+ if (encoded.charAt(1) != '|') {
+ return encoded;
+ } // not encoded
+ switch (encoded.charAt(0)) {
+ case 'b': {
+ return
kelondroBase64Order.enhancedCoder.decodeString(encoded.substring(2),
"de.anomic.tools.crypt.simpleDecode()");
+ }
+ case 'z':
+ try {
+ return
gzip.gunzipString(kelondroBase64Order.enhancedCoder.decode(encoded.substring(2),
"de.anomic.tools.crypt.simpleDecode()"));
+ } catch (Exception e) {
+ e.printStackTrace();
+ return null;
+ }
+ case 'p': {
+ return encoded.substring(2);
+ }
+ default: {
+ return null;
+ }
+ }
}
- }
public static void main(String[] args) {
String
teststring="1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
Modified: trunk/source/de/anomic/tools/cryptbig.java
===================================================================
--- trunk/source/de/anomic/tools/cryptbig.java 2008-02-27 20:09:22 UTC (rev
4518)
+++ trunk/source/de/anomic/tools/cryptbig.java 2008-02-27 23:12:43 UTC (rev
4519)
@@ -395,7 +395,7 @@
return salt + ((gzFlag) ? "1" : "0") +
kelondroBase64Order.enhancedCoder.encode(enc);
}
- public static String descrambleString(String key, String s) {
+ public static String descrambleString(String key, String s) throws
IOException {
String salt = s.substring(0, 8);
boolean gzFlag = (s.charAt(8) == '1');
s = s.substring(9);
@@ -434,7 +434,7 @@
return null;
}
- public static String simpleDecode(String encoded, String key) {
+ public static String simpleDecode(String encoded, String key) throws
IOException {
if ((encoded == null) || (encoded.length() < 3)) return null;
if (encoded.charAt(1) != '|') return encoded; // not encoded
char method = encoded.charAt(0);
@@ -620,7 +620,12 @@
// 'descramble' string
if (s.length != 3) {help(); System.exit(-1);}
long t = System.currentTimeMillis();
- System.out.println(descrambleString(s[1], s[2]));
+ try {
+ System.out.println(descrambleString(s[1], s[2]));
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
System.out.println("Calculation time: " +
(System.currentTimeMillis() - t) + " milliseconds");
System.exit(0);
}
Modified: trunk/source/de/anomic/tools/gzip.java
===================================================================
--- trunk/source/de/anomic/tools/gzip.java 2008-02-27 20:09:22 UTC (rev
4518)
+++ trunk/source/de/anomic/tools/gzip.java 2008-02-27 23:12:43 UTC (rev
4519)
@@ -107,19 +107,13 @@
}
}
- public static String gunzipString(byte[] in) {
- try {
+ public static String gunzipString(byte[] in) throws IOException {
InputStream fin = new GZIPInputStream(new
ByteArrayInputStream(in));
ByteArrayOutputStream fout = new ByteArrayOutputStream();
copy(fout, fin, 128);
fin.close();
fout.close();
return new String(fout.toByteArray(), "UTF-8");
- } catch (IOException e) {
- //System.err.println("ERROR: IO trouble ");
- logger.logWarning("ERROR: IO trouble ",e);
- return null;
- }
}
private static void copy(OutputStream out, InputStream in, int bufferSize)
throws IOException {
Modified: trunk/source/de/anomic/yacy/yacyClient.java
===================================================================
--- trunk/source/de/anomic/yacy/yacyClient.java 2008-02-27 20:09:22 UTC (rev
4518)
+++ trunk/source/de/anomic/yacy/yacyClient.java 2008-02-27 23:12:43 UTC (rev
4519)
@@ -233,9 +233,15 @@
yacyCore.log.logInfo("hello/client: rejected contacting seed;
too large (" + seedStr.length() + " > " + yacySeed.maxsize + ")");
} else {
//System.out.println("DEBUG yacyClient.publishMySeed seedStr =
" + seedStr);
- if
(yacyCore.peerActions.peerArrival(yacySeed.genRemoteSeed(seedStr,
post.get("key", ""), true), (i == 1))) count++;
+ yacySeed remoteSeed = yacySeed.genRemoteSeed(seedStr,
post.get("key", ""), true);
+ if (remoteSeed == null) {
+ yacyCore.log.logWarning("hello/client: bad seed string
from peer " + otherHash + ", address = " + address + ", count = " + (i-1) + "
seedStr = " + seedStr);
+ } else {
+ if (yacyCore.peerActions.peerArrival(remoteSeed, (i ==
1))) count++;
+ }
}
}
+
return count;
}
_______________________________________________
YaCy-svn mailing list
YaCy-svn@lists.berlios.de
https://lists.berlios.de/mailman/listinfo/yacy-svn
