Author: orbiter Date: 2008-02-03 21:11:50 +0100 (Sun, 03 Feb 2008) New Revision: 4438
Modified: trunk/build.properties trunk/htroot/PerformanceQueues_p.html trunk/htroot/PerformanceQueues_p.java trunk/htroot/index.java trunk/htroot/yacysearch.java trunk/source/de/anomic/server/serverSystem.java trunk/yacy.init Log: - fix for online caution delay settings, see http://forum.yacy-websuche.de/viewtopic.php?f=6&t=738&p=4723#p4723 - removed remote search limitation for non-dht-peers according to discussion in http://forum.yacy-websuche.de/viewtopic.php?f=15&t=793&hilit=&p=5277#p5277 Modified: trunk/build.properties =================================================================== --- trunk/build.properties 2008-02-03 18:42:25 UTC (rev 4437) +++ trunk/build.properties 2008-02-03 20:11:50 UTC (rev 4438) @@ -3,7 +3,7 @@ javacTarget=1.5 # Release Configuration -releaseVersion=0.566 +releaseVersion=0.567 stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz Modified: trunk/htroot/PerformanceQueues_p.html =================================================================== --- trunk/htroot/PerformanceQueues_p.html 2008-02-03 18:42:25 UTC (rev 4437) +++ trunk/htroot/PerformanceQueues_p.html 2008-02-03 20:11:50 UTC (rev 4438) @@ -205,14 +205,20 @@ <form action="PerformanceQueues_p.html" method="post" enctype="multipart/form-data"> <fieldset><legend>Proxy Performance Settings</legend> <p> - This is the time that the crawler idles when the proxy is accessed. The delay is extended by this time - each time the proxy is accessed afterwards. This shall improve performance of the proxy throughput. - (current delta is #[onlineCautionDelayCurrent]# milliseconds since last proxy access.) + This is the time that the crawler idles when the proxy is accessed, or a local or remote search is done. + The delay is extended by this time each time the proxy is accessed afterwards. + This shall improve performance of the affected process (proxy or search). + (current delta is #[crawlPauseProxyCurrent]#/#[crawlPauseLocalsearchCurrent]#/#[crawlPauseRemotesearchCurrent]# + seconds since last proxy/local-search/remote-search access.) </p> <dl> - <dt><label for="onlineCautionDelay">Online Caution Delay (milliseconds)</label>:</dt> - <dd><input id="onlineCautionDelay" name="onlineCautionDelay" type="text" size="20" maxlength="100" value="#[onlineCautionDelay]#" /></dd> - <dt><input type="submit" name="proxyControlSubmit" value="Enter New Parameters" /></dt> + <dt><label for="crawlPauseProxy">Online Caution Delay for Proxy (milliseconds)</label>:</dt> + <dd><input id="crawlPauseProxy" name="crawlPauseProxy" type="text" size="20" maxlength="100" value="#[crawlPauseProxy]#" /></dd> + <dt><label for="crawlPauseLocalsearch">Online Caution Delay for Local Search (milliseconds)</label>:</dt> + <dd><input id="crawlPauseLocalsearch" name="crawlPauseLocalsearch" type="text" size="20" maxlength="100" value="#[crawlPauseLocalsearch]#" /></dd> + <dt><label for="crawlPauseRemotesearch">Online Caution Delay for Remote Search (milliseconds)</label>:</dt> + <dd><input id="crawlPauseRemotesearch" name="crawlPauseRemotesearch" type="text" size="20" maxlength="100" value="#[crawlPauseRemotesearch]#" /></dd> + <dt><input type="submit" name="onlineCautionSubmit" value="Enter New Parameters" /></dt> <dd>Changes take effect immediately</dd> </dl> </fieldset> Modified: trunk/htroot/PerformanceQueues_p.java =================================================================== --- trunk/htroot/PerformanceQueues_p.java 2008-02-03 18:42:25 UTC (rev 4437) +++ trunk/htroot/PerformanceQueues_p.java 2008-02-03 20:11:50 UTC (rev 4438) @@ -227,9 +227,10 @@ switchboard.setConfig("javastart_priority",post.get("YaCyPriority","0")); } - if ((post != null) && (post.containsKey("proxyControlSubmit"))) { - int onlineCautionDelay = post.getInt("onlineCautionDelay", 30000); - switchboard.setConfig("onlineCautionDelay", Integer.toString(onlineCautionDelay)); + if ((post != null) && (post.containsKey("onlineCautionSubmit"))) { + switchboard.setConfig(plasmaSwitchboard.PROXY_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseProxy", 30000))); + switchboard.setConfig(plasmaSwitchboard.LOCALSEACH_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseLocalsearch", 30000))); + switchboard.setConfig(plasmaSwitchboard.REMOTESEARCH_ONLINE_CAUTION_DELAY, Integer.toString(post.getInt("crawlPauseRemotesearch", 30000))); } // table cache settings @@ -248,8 +249,12 @@ prop.put("wordCacheMaxCount", switchboard.getConfigLong(plasmaSwitchboard.WORDCACHE_MAX_COUNT, 20000)); prop.put("wordCacheInitCount", switchboard.getConfigLong(plasmaSwitchboard.WORDCACHE_INIT_COUNT, 30000)); prop.put("wordFlushSize", switchboard.getConfigLong("wordFlushSize", 2000)); - prop.put("onlineCautionDelay", switchboard.getConfigLong("onlineCautionDelay", 30000)); - prop.putNum("onlineCautionDelayCurrent", System.currentTimeMillis() - switchboard.proxyLastAccess); + prop.put("crawlPauseProxy", switchboard.getConfigLong(plasmaSwitchboard.PROXY_ONLINE_CAUTION_DELAY, 30000)); + prop.put("crawlPauseLocalsearch", switchboard.getConfigLong(plasmaSwitchboard.LOCALSEACH_ONLINE_CAUTION_DELAY, 30000)); + prop.put("crawlPauseRemotesearch", switchboard.getConfigLong(plasmaSwitchboard.REMOTESEARCH_ONLINE_CAUTION_DELAY, 30000)); + prop.putNum("crawlPauseProxyCurrent", (System.currentTimeMillis() - switchboard.proxyLastAccess) / 1000); + prop.putNum("crawlPauseLocalsearchCurrent", (System.currentTimeMillis() - switchboard.localSearchLastAccess) / 1000); + prop.putNum("crawlPauseRemotesearchCurrent", (System.currentTimeMillis() - switchboard.remoteSearchLastAccess) / 1000); // table thread pool settings prop.put("pool_0_name","Crawler Pool"); Modified: trunk/htroot/index.java =================================================================== --- trunk/htroot/index.java 2008-02-03 18:42:25 UTC (rev 4437) +++ trunk/htroot/index.java 2008-02-03 20:11:50 UTC (rev 4438) @@ -68,26 +68,8 @@ final boolean indexDistributeGranted = sb.getConfigBool(plasmaSwitchboard.INDEX_DIST_ALLOW, true); final boolean indexReceiveGranted = sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, true); - global = global && indexDistributeGranted && indexReceiveGranted; -/* - final String referer = (String) header.get(httpHeader.REFERER); - if (referer != null) { - yacyURL url; - try { - url = new yacyURL(referer, null); - } catch (MalformedURLException e) { - url = null; - } - if ((url != null) && (!url.isLocal())) { - final HashMap referrerprop = new HashMap(); - referrerprop.put("count", "1"); - referrerprop.put("clientip", header.get(httpHeader.CONNECTION_PROP_CLIENTIP)); - referrerprop.put("useragent", header.get(httpHeader.USER_AGENT)); - referrerprop.put("date", (new serverDate()).toShortString(false)); - if (sb.facilityDB != null) try {sb.facilityDB.update("backlinks", referer, referrerprop);} catch (IOException e) {} - } - } -*/ + //global = global && indexDistributeGranted && indexReceiveGranted; + // search domain int contentdom = plasmaSearchQuery.CONTENTDOM_TEXT; String cds = (post == null) ? "text" : post.get("contentdom", "text"); Modified: trunk/htroot/yacysearch.java =================================================================== --- trunk/htroot/yacysearch.java 2008-02-03 18:42:25 UTC (rev 4437) +++ trunk/htroot/yacysearch.java 2008-02-03 20:11:50 UTC (rev 4438) @@ -91,22 +91,6 @@ boolean rss = (post == null) ? false : post.get("rss", "false").equals("true"); if ((post == null) || (env == null) || (querystring.length() == 0) || (!searchAllowed)) { - /* - // save referrer - final String referer = (String) header.get("Referer"); - if (referer != null) { - yacyURL url; - try { url = new yacyURL(referer, null); } catch (MalformedURLException e) { url = null; } - if ((url != null) && (!url.isLocal())) { - final HashMap referrerprop = new HashMap(); - referrerprop.put("count", "1"); - referrerprop.put("clientip", header.get("CLIENTIP")); - referrerprop.put("useragent", header.get("User-Agent")); - referrerprop.put("date", (new serverDate()).toShortString(false)); - if (sb.facilityDB != null) try { sb.facilityDB.update("backlinks", referer, referrerprop); } catch (IOException e) {} - } - } - */ // we create empty entries for template strings final serverObjects prop = new serverObjects(); prop.put("searchagain", "0"); @@ -169,13 +153,13 @@ } // SEARCH - final boolean indexDistributeGranted = sb.getConfig(plasmaSwitchboard.INDEX_DIST_ALLOW, "true").equals("true"); - final boolean indexReceiveGranted = sb.getConfig("allowReceiveIndex", "true").equals("true"); - final boolean offline = yacyCore.seedDB.mySeed().isVirgin(); + //final boolean indexDistributeGranted = sb.getConfig(plasmaSwitchboard.INDEX_DIST_ALLOW, "true").equals("true"); + //final boolean indexReceiveGranted = sb.getConfig("allowReceiveIndex", "true").equals("true"); + //final boolean offline = yacyCore.seedDB.mySeed().isVirgin(); final boolean clustersearch = sb.isRobinsonMode() && (sb.getConfig("cluster.mode", "").equals("privatecluster") || sb.getConfig("cluster.mode", "").equals("publiccluster")); - if (offline || !indexDistributeGranted || !indexReceiveGranted) { global = false; } + //if (offline || !indexDistributeGranted || !indexReceiveGranted) { global = false; } if (clustersearch) global = true; // switches search on, but search target is limited to cluster nodes // find search domain Modified: trunk/source/de/anomic/server/serverSystem.java =================================================================== --- trunk/source/de/anomic/server/serverSystem.java 2008-02-03 18:42:25 UTC (rev 4437) +++ trunk/source/de/anomic/server/serverSystem.java 2008-02-03 20:11:50 UTC (rev 4438) @@ -50,6 +50,8 @@ import java.util.Properties; import java.util.Vector; +import de.anomic.server.logging.serverLog; + public final class serverSystem { // constants for system identification @@ -330,6 +332,7 @@ try { Runtime.getRuntime().exec("chmod 755 " + scriptFile.getAbsolutePath().replaceAll(" ", "\\ ")).waitFor(); } catch (InterruptedException e) { + serverLog.logSevere("DEPLOY", "deploy of script file failed. file = " + scriptFile.getAbsolutePath(), e); throw new IOException(e.getMessage()); } } Modified: trunk/yacy.init =================================================================== --- trunk/yacy.init 2008-02-03 18:42:25 UTC (rev 4437) +++ trunk/yacy.init 2008-02-03 20:11:50 UTC (rev 4438) @@ -725,11 +725,11 @@ crawler.http.acceptLanguage=en-us,en;q=0.5 crawler.http.acceptCharset=ISO-8859-1,utf-8;q=0.7,*;q=0.7 crawler.http.maxFileSize=262144 -crawler.http.maxFileSize__pro=1048576 +crawler.http.maxFileSize__pro=262144 # ftp crawler specific settings; size in bytes crawler.ftp.maxFileSize=262144 -crawler.ftp.maxFileSize__pro=1048576 +crawler.ftp.maxFileSize__pro=262144 # maximum number of crawler threads crawler.MaxActiveThreads = 30 @@ -903,27 +903,6 @@ publicSurftips = true publicSearchpage = true -# a Java Properties file containig a list of SOAP services that should deployed -# additionally to the default services. E.g. -# soap.serviceDeploymentList = DATA/SETTINGS/myServices.properties -# -# One entry in the property file should must have the following format: -# servicename=fullClassname -# -# e.g. -# test=org.myservices.test -# -# Servicename is the name that shoud be used to access the service, e.g. -# if the service name is "test" then the service can be reached using -# http://localhost:8080/soap/test -# -# The WSDL document that belongs to the deployed service could be reached -# using -# http://localhost:8080/soap/test?wsdl -# -soap.serviceDeploymentList = - - # Wiki access rights # the built-in wiki system allows by default only that the administrator is allowed to make changes # this can be changed. There are three options: _______________________________________________ YaCy-svn mailing list YaCy-svn@lists.berlios.de https://lists.berlios.de/mailman/listinfo/yacy-svn
