Author: orbiter
Date: 2008-03-15 01:49:16 +0100 (Sat, 15 Mar 2008)
New Revision: 4562
Modified:
trunk/htroot/xml/bookmarks/xbel/
trunk/htroot/yacy/ui/
trunk/source/de/anomic/http/httpc.java
trunk/source/de/anomic/http/httpdByteCountInputStream.java
trunk/source/de/anomic/plasma/plasmaCrawlBalancer.java
trunk/source/de/anomic/plasma/plasmaCrawlNURL.java
trunk/source/de/anomic/plasma/plasmaSearchEvent.java
trunk/source/de/anomic/plasma/plasmaSwitchboard.java
trunk/source/de/anomic/plasma/plasmaWordIndex.java
trunk/source/de/anomic/server/serverCore.java
trunk/source/de/anomic/yacy/yacyClient.java
Log:
fixed deadlock
Property changes on: trunk/htroot/xml/bookmarks/xbel
___________________________________________________________________
Name: svn:ignore
+ *.class
Property changes on: trunk/htroot/yacy/ui
___________________________________________________________________
Name: svn:ignore
+ *.class
Modified: trunk/source/de/anomic/http/httpc.java
===================================================================
--- trunk/source/de/anomic/http/httpc.java 2008-03-14 16:28:27 UTC (rev
4561)
+++ trunk/source/de/anomic/http/httpc.java 2008-03-15 00:49:16 UTC (rev
4562)
@@ -395,12 +395,14 @@
// trying to establish a connection to the address
this.initTime = System.currentTimeMillis();
this.lastIO = System.currentTimeMillis();
- this.socket.setKeepAlive(false);
+ this.socket.setKeepAlive(true);
// set socket timeout and keep alive behavior
assert timeout >= 1000;
this.socket.setSoTimeout(timeout); // waiting time for read
this.socket.setTcpNoDelay(true); // no accumulation until buffer
is full
this.socket.setSoLinger(true, timeout); // wait for all data being
written on close()
+ this.socket.setSendBufferSize(1440); // read
http://www.cisco.com/warp/public/105/38.shtml
+ this.socket.setReceiveBufferSize(1440); // read
http://www.cisco.com/warp/public/105/38.shtml
// get the connection
this.socket.connect(address, timeout);
@@ -1411,8 +1413,9 @@
*
* @param procOS
* @param file
+ * @throws IOException
*/
- public void writeContent(Object procOS, File file) {
+ public void writeContent(Object procOS, File file) throws IOException {
// this writes the input stream to either another output stream or
// a file or both.
FileOutputStream bufferOS = null;
@@ -1421,22 +1424,18 @@
} catch (FileNotFoundException e) {
file = null;
}
- try {
- InputStream is = this.getContentInputStream();
- if (procOS == null) {
- writeX(is, null, bufferOS);
- } else if (procOS instanceof OutputStream) {
- writeX(is, (OutputStream) procOS, bufferOS);
- //writeContentX(httpc.this.clientInput, this.gzip,
this.responseHeader.contentLength(), procOS, bufferOS);
- } else if (procOS instanceof Writer) {
- String charSet =
this.responseHeader.getCharacterEncoding();
- if (charSet == null) charSet = httpHeader.DEFAULT_CHARSET;
- writeX(is, charSet, (Writer) procOS, bufferOS, charSet);
- } else {
- throw new IllegalArgumentException("Invalid procOS object
type '" + procOS.getClass().getName() + "'");
- }
- } catch (IOException e) {
- e.printStackTrace();
+ InputStream is = this.getContentInputStream();
+ if (procOS == null) {
+ writeX(is, null, bufferOS);
+ } else if (procOS instanceof OutputStream) {
+ writeX(is, (OutputStream) procOS, bufferOS);
+ //writeContentX(httpc.this.clientInput, this.gzip,
this.responseHeader.contentLength(), procOS, bufferOS);
+ } else if (procOS instanceof Writer) {
+ String charSet = this.responseHeader.getCharacterEncoding();
+ if (charSet == null) charSet = httpHeader.DEFAULT_CHARSET;
+ writeX(is, charSet, (Writer) procOS, bufferOS, charSet);
+ } else {
+ throw new IllegalArgumentException("Invalid procOS object type
'" + procOS.getClass().getName() + "'");
}
if (bufferOS != null) {
@@ -1451,12 +1450,12 @@
}
- public void writeX(InputStream source, OutputStream procOS,
OutputStream bufferOS) {
+ public void writeX(InputStream source, OutputStream procOS,
OutputStream bufferOS) throws IOException {
byte[] buffer = new byte[2048];
int l, c = 0;
lastIO = System.currentTimeMillis();
- io: while (true) try {
+ io: while (true) {
l = source.read(buffer, 0, buffer.length);
if (l < 0) break;
if (l == 0) try {
@@ -1470,10 +1469,6 @@
c += l;
if (procOS != null) procOS.write(buffer, 0, l);
if (bufferOS != null) bufferOS.write(buffer, 0, l);
- } catch (IOException e) {
- System.out.println("*** DEBUG: writeX/IOStream terminated with
IOException, processed " + c + " bytes. cause: " + e.getMessage());
- e.printStackTrace();
- break;
}
// flush the streams
Modified: trunk/source/de/anomic/http/httpdByteCountInputStream.java
===================================================================
--- trunk/source/de/anomic/http/httpdByteCountInputStream.java 2008-03-14
16:28:27 UTC (rev 4561)
+++ trunk/source/de/anomic/http/httpdByteCountInputStream.java 2008-03-15
00:49:16 UTC (rev 4562)
@@ -96,7 +96,6 @@
if (readCount > 0) this.byteCount += readCount;
return readCount;
} catch (IOException e) {
- e.printStackTrace();
throw new IOException(e.getMessage() + "; b.length = " + b.length
+ ", off = " + off + ", len = " + len);
}
}
Modified: trunk/source/de/anomic/plasma/plasmaCrawlBalancer.java
===================================================================
--- trunk/source/de/anomic/plasma/plasmaCrawlBalancer.java 2008-03-14
16:28:27 UTC (rev 4561)
+++ trunk/source/de/anomic/plasma/plasmaCrawlBalancer.java 2008-03-15
00:49:16 UTC (rev 4562)
@@ -118,6 +118,7 @@
public synchronized void close() {
while (domainStacksNotEmpty()) flushOnceDomStacks(0, true); // flush
to ram, because the ram flush is optimized
+ size();
try { flushAllRamStack(); } catch (IOException e) {}
if (urlFileIndex != null) {
urlFileIndex.close();
Modified: trunk/source/de/anomic/plasma/plasmaCrawlNURL.java
===================================================================
--- trunk/source/de/anomic/plasma/plasmaCrawlNURL.java 2008-03-14 16:28:27 UTC
(rev 4561)
+++ trunk/source/de/anomic/plasma/plasmaCrawlNURL.java 2008-03-15 00:49:16 UTC
(rev 4562)
@@ -62,9 +62,9 @@
public static final int STACK_TYPE_MOVIE = 12; // put on movie stack
public static final int STACK_TYPE_MUSIC = 13; // put on music stack
- private static final long minimumLocalDelta = 50; // the minimum time
difference between access of the same local domain
- private static final long minimumGlobalDelta = 500; // the minimum time
difference between access of the same global domain
- private static final long maximumDomAge = 60000; // the maximum age of a
domain until it is used for another crawl attempt
+ private static final long minimumLocalDelta = 10; // the minimum time
difference between access of the same local domain
+ private static final long minimumGlobalDelta = 333; // the minimum time
difference between access of the same global domain
+ private static final long maximumDomAge = 60000; // the maximum age of
a domain until it is used for another crawl attempt
private plasmaCrawlBalancer coreStack; // links found by crawling to
depth-1
private plasmaCrawlBalancer limitStack; // links found by crawling at
target depth
Modified: trunk/source/de/anomic/plasma/plasmaSearchEvent.java
===================================================================
--- trunk/source/de/anomic/plasma/plasmaSearchEvent.java 2008-03-14
16:28:27 UTC (rev 4561)
+++ trunk/source/de/anomic/plasma/plasmaSearchEvent.java 2008-03-15
00:49:16 UTC (rev 4562)
@@ -245,9 +245,7 @@
// sort the local containers and truncate it to a limited count,
// so following sortings together with the global results will be
fast
- synchronized (rankedCache) {
- rankedCache.execQuery();
- }
+ rankedCache.execQuery();
}
}
Modified: trunk/source/de/anomic/plasma/plasmaSwitchboard.java
===================================================================
--- trunk/source/de/anomic/plasma/plasmaSwitchboard.java 2008-03-14
16:28:27 UTC (rev 4561)
+++ trunk/source/de/anomic/plasma/plasmaSwitchboard.java 2008-03-15
00:49:16 UTC (rev 4562)
@@ -1719,6 +1719,7 @@
log.logConfig("SWITCHBOARD SHUTDOWN STEP 2: sending termination signal
to threaded indexing");
// closing all still running db importer jobs
this.dbImportManager.close();
+ httpc.closeAllConnections();
crawlQueues.close();
wikiDB.close();
blogDB.close();
@@ -1733,7 +1734,6 @@
parser.close();
plasmaHTCache.close();
sbQueue.close();
- httpc.closeAllConnections();
webStructure.flushCitationReference("crg");
webStructure.close();
log.logConfig("SWITCHBOARD SHUTDOWN STEP 3: sending termination signal
to database manager (stand by...)");
Modified: trunk/source/de/anomic/plasma/plasmaWordIndex.java
===================================================================
--- trunk/source/de/anomic/plasma/plasmaWordIndex.java 2008-03-14 16:28:27 UTC
(rev 4561)
+++ trunk/source/de/anomic/plasma/plasmaWordIndex.java 2008-03-15 00:49:16 UTC
(rev 4562)
@@ -340,24 +340,18 @@
// get from cache
indexContainer container;
- synchronized (dhtOutCache) {
- container = dhtOutCache.getContainer(wordHash, urlselection);
+ container = dhtOutCache.getContainer(wordHash, urlselection);
+ if (container == null) {
+ container = dhtInCache.getContainer(wordHash, urlselection);
+ } else {
+ container.addAllUnique(dhtInCache.getContainer(wordHash,
urlselection));
}
- synchronized (dhtInCache) {
- if (container == null) {
- container = dhtInCache.getContainer(wordHash,
urlselection);
- } else {
-
container.addAllUnique(dhtInCache.getContainer(wordHash, urlselection));
- }
- }
// get from collection index
- synchronized (this) {
- if (container == null) {
- container = collections.getContainer(wordHash, urlselection);
- } else {
- container.addAllUnique(collections.getContainer(wordHash,
urlselection));
- }
+ if (container == null) {
+ container = collections.getContainer(wordHash, urlselection);
+ } else {
+ container.addAllUnique(collections.getContainer(wordHash,
urlselection));
}
if (container == null) return null;
@@ -443,10 +437,8 @@
public void close() {
dhtInCache.close();
dhtOutCache.close();
- synchronized (this) {
- collections.close();
- loadedURL.close();
- }
+ collections.close();
+ loadedURL.close();
}
public indexContainer deleteContainer(String wordHash) {
@@ -455,29 +447,17 @@
indexRWIRowEntry.urlEntryRow,
dhtInCache.sizeContainer(wordHash) +
dhtOutCache.sizeContainer(wordHash) + collections.indexSize(wordHash)
);
- synchronized (dhtInCache) {
- c.addAllUnique(dhtInCache.deleteContainer(wordHash));
- }
- synchronized (dhtOutCache) {
- c.addAllUnique(dhtOutCache.deleteContainer(wordHash));
- }
- synchronized (this) {
- c.addAllUnique(collections.deleteContainer(wordHash));
- }
+ c.addAllUnique(dhtInCache.deleteContainer(wordHash));
+ c.addAllUnique(dhtOutCache.deleteContainer(wordHash));
+ c.addAllUnique(collections.deleteContainer(wordHash));
return c;
}
public boolean removeEntry(String wordHash, String urlHash) {
boolean removed = false;
- synchronized (dhtInCache) {
- removed = removed | (dhtInCache.removeEntry(wordHash, urlHash));
- }
- synchronized (dhtOutCache) {
- removed = removed | (dhtOutCache.removeEntry(wordHash,
urlHash));
- }
- synchronized (this) {
- removed = removed | (collections.removeEntry(wordHash, urlHash));
- }
+ removed = removed | (dhtInCache.removeEntry(wordHash, urlHash));
+ removed = removed | (dhtOutCache.removeEntry(wordHash, urlHash));
+ removed = removed | (collections.removeEntry(wordHash, urlHash));
return removed;
}
@@ -494,29 +474,17 @@
public int removeEntries(String wordHash, Set<String> urlHashes) {
int removed = 0;
- synchronized (dhtInCache) {
- removed += dhtInCache.removeEntries(wordHash, urlHashes);
- }
- synchronized (dhtOutCache) {
- removed += dhtOutCache.removeEntries(wordHash, urlHashes);
- }
- synchronized (this) {
- removed += collections.removeEntries(wordHash, urlHashes);
- }
+ removed += dhtInCache.removeEntries(wordHash, urlHashes);
+ removed += dhtOutCache.removeEntries(wordHash, urlHashes);
+ removed += collections.removeEntries(wordHash, urlHashes);
return removed;
}
public String removeEntriesExpl(String wordHash, Set<String> urlHashes) {
String removed = "";
- synchronized (dhtInCache) {
- removed += dhtInCache.removeEntries(wordHash, urlHashes) + ", ";
- }
- synchronized (dhtOutCache) {
- removed += dhtOutCache.removeEntries(wordHash, urlHashes) + ", ";
- }
- synchronized (this) {
- removed += collections.removeEntries(wordHash, urlHashes);
- }
+ removed += dhtInCache.removeEntries(wordHash, urlHashes) + ", ";
+ removed += dhtOutCache.removeEntries(wordHash, urlHashes) + ", ";
+ removed += collections.removeEntries(wordHash, urlHashes);
return removed;
}
Modified: trunk/source/de/anomic/server/serverCore.java
===================================================================
--- trunk/source/de/anomic/server/serverCore.java 2008-03-14 16:28:27 UTC
(rev 4561)
+++ trunk/source/de/anomic/server/serverCore.java 2008-03-15 00:49:16 UTC
(rev 4562)
@@ -423,7 +423,9 @@
controlSocket.setSoLinger(true, this.timeout);
// ensure that MTU-48 is not exceeded to prevent that routers
cannot handle large data packets
+ // read http://www.cisco.com/warp/public/105/38.shtml for
explanation
controlSocket.setSendBufferSize(1440);
+ controlSocket.setReceiveBufferSize(1440);
// create session
Session connection = new Session(sessionThreadGroup,
controlSocket, this.timeout);
Modified: trunk/source/de/anomic/yacy/yacyClient.java
===================================================================
--- trunk/source/de/anomic/yacy/yacyClient.java 2008-03-14 16:28:27 UTC (rev
4561)
+++ trunk/source/de/anomic/yacy/yacyClient.java 2008-03-15 00:49:16 UTC (rev
4562)
@@ -99,7 +99,7 @@
HashMap<String, String> result = null;
final serverObjects post =
yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), null);
- for (int retry = 0; retry < 3; retry++) try {
+ for (int retry = 0; retry < 4; retry++) try {
// generate request
post.put("count", "20");
post.put("seed",
yacyCore.seedDB.mySeed().genSeedStr(post.get("key", "")));
@@ -119,10 +119,10 @@
break;
} catch (Exception e) {
if (Thread.currentThread().isInterrupted()) {
- yacyCore.log.logFine("yacyClient.publishMySeed thread '" +
Thread.currentThread().getName() + "' interrupted.");
+ yacyCore.log.logWarning("yacyClient.publishMySeed thread '" +
Thread.currentThread().getName() + "' interrupted.");
return -1;
} else {
- yacyCore.log.logFine("yacyClient.publishMySeed thread '" +
Thread.currentThread().getName() + "' exception: " + e.getMessage() + "; retry
= " + retry); // here VERY OFTEN a 'Connection reset' appears. What is the
cause?
+ yacyCore.log.logWarning("yacyClient.publishMySeed thread '" +
Thread.currentThread().getName() + "' exception: " + e.getMessage() + "; retry
= " + retry); // here VERY OFTEN a 'Connection reset' appears. What is the
cause?
// try again (go into loop)
}
result = null;
_______________________________________________
YaCy-svn mailing list
[email protected]
https://lists.berlios.de/mailman/listinfo/yacy-svn
