Author: orbiter
Date: 2008-02-01 00:40:47 +0100 (Fri, 01 Feb 2008)
New Revision: 4424
Modified:
trunk/build.properties
trunk/source/de/anomic/http/httpdSwitchboard.java
trunk/source/de/anomic/index/indexContainer.java
trunk/source/de/anomic/index/indexRWIEntry.java
trunk/source/de/anomic/index/indexRWIRowEntry.java
trunk/source/de/anomic/index/indexRWIVarEntry.java
trunk/source/de/anomic/plasma/plasmaCrawlNURL.java
trunk/source/de/anomic/plasma/plasmaSearchEvent.java
trunk/source/de/anomic/plasma/plasmaSwitchboard.java
trunk/source/de/anomic/server/logging/serverLog.java
trunk/source/de/anomic/server/serverAbstractSwitch.java
trunk/source/de/anomic/server/serverPlainSwitch.java
trunk/source/de/anomic/server/serverSwitch.java
trunk/source/de/anomic/yacy/yacyClient.java
trunk/source/de/anomic/yacy/yacyVersion.java
trunk/source/migration.java
trunk/source/yacy.java
Log:
- fixed bug with too much RAM in crawler queue
- fixed dir bug
- better calculation of TF for join
- better waiting-on-result logic
Modified: trunk/build.properties
===================================================================
--- trunk/build.properties 2008-01-31 16:43:34 UTC (rev 4423)
+++ trunk/build.properties 2008-01-31 23:40:47 UTC (rev 4424)
@@ -3,7 +3,7 @@
javacTarget=1.5
# Release Configuration
-releaseVersion=0.564
+releaseVersion=0.565
stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
Modified: trunk/source/de/anomic/http/httpdSwitchboard.java
===================================================================
--- trunk/source/de/anomic/http/httpdSwitchboard.java 2008-01-31 16:43:34 UTC
(rev 4423)
+++ trunk/source/de/anomic/http/httpdSwitchboard.java 2008-01-31 23:40:47 UTC
(rev 4424)
@@ -40,6 +40,7 @@
package de.anomic.http;
+import java.io.File;
import java.util.LinkedList;
import de.anomic.server.serverAbstractSwitch;
@@ -50,7 +51,7 @@
private final LinkedList<Object> cacheStack;
- public httpdSwitchboard(String rootPath, String initPath, String
configPath, boolean applyPro) {
+ public httpdSwitchboard(File rootPath, String initPath, String configPath,
boolean applyPro) {
super(rootPath, initPath, configPath, applyPro);
cacheStack = new LinkedList<Object>();
}
Modified: trunk/source/de/anomic/index/indexContainer.java
===================================================================
--- trunk/source/de/anomic/index/indexContainer.java 2008-01-31 16:43:34 UTC
(rev 4423)
+++ trunk/source/de/anomic/index/indexContainer.java 2008-01-31 23:40:47 UTC
(rev 4424)
@@ -325,7 +325,7 @@
assert (ie0.urlHash().length() == keylength) :
"ie0.urlHash() = " + ie0.urlHash();
assert (ie1.urlHash().length() == keylength) :
"ie1.urlHash() = " + ie1.urlHash();
// this is a hit. Calculate word distance:
- ie0.combineDistance(ie1);
+ ie0.join(ie1);
if (ie0.worddistance() <= maxDistance) conj.add(ie0);
}
}
@@ -360,7 +360,7 @@
if (e2.hasNext()) ie2 = (indexRWIEntry) e2.next(); else
break;
} else {
// we have found the same urls in different searches!
- ie1.combineDistance(ie2);
+ ie1.join(ie2);
if (ie1.worddistance() <= maxDistance) conj.add(ie1);
if (e1.hasNext()) ie1 = (indexRWIEntry) e1.next(); else
break;
if (e2.hasNext()) ie2 = (indexRWIEntry) e2.next(); else
break;
@@ -435,7 +435,7 @@
if (e2.hasNext()) ie2 = (indexRWIEntry) e2.next(); else
break;
} else {
// we have found the same urls in different searches!
- ie1.combineDistance(ie2);
+ ie1.join(ie2);
e1.remove();
if (e1.hasNext()) ie1 = (indexRWIEntry) e1.next(); else
break;
if (e2.hasNext()) ie2 = (indexRWIEntry) e2.next(); else
break;
Modified: trunk/source/de/anomic/index/indexRWIEntry.java
===================================================================
--- trunk/source/de/anomic/index/indexRWIEntry.java 2008-01-31 16:43:34 UTC
(rev 4423)
+++ trunk/source/de/anomic/index/indexRWIEntry.java 2008-01-31 23:40:47 UTC
(rev 4424)
@@ -87,7 +87,7 @@
public String toString();
- public void combineDistance(indexRWIEntry oe);
+ public void join(indexRWIEntry oe);
public int worddistance();
Modified: trunk/source/de/anomic/index/indexRWIRowEntry.java
===================================================================
--- trunk/source/de/anomic/index/indexRWIRowEntry.java 2008-01-31 16:43:34 UTC
(rev 4423)
+++ trunk/source/de/anomic/index/indexRWIRowEntry.java 2008-01-31 23:40:47 UTC
(rev 4424)
@@ -263,18 +263,22 @@
return toPropertyForm();
}
- public static indexRWIEntry combineDistance(indexRWIRowEntry ie1,
indexRWIEntry ie2) {
+ public static indexRWIEntry join(indexRWIRowEntry ie1, indexRWIEntry ie2) {
// returns a modified entry of the first argument
+
+ // combine the distance
ie1.entry.setCol(col_worddistance, ie1.worddistance() +
ie2.worddistance() + Math.abs(ie1.posintext() - ie2.posintext()));
ie1.entry.setCol(col_posintext, Math.min(ie1.posintext(),
ie2.posintext()));
- ie1.entry.setCol(col_posinphrase, (ie1.posofphrase() ==
ie2.posofphrase()) ? ie1.posofphrase() : 0 /*unknown*/);
+ ie1.entry.setCol(col_posinphrase, (ie1.posofphrase() ==
ie2.posofphrase()) ? Math.min(ie1.posinphrase(), ie2.posinphrase()) : 0
/*unknown*/);
ie1.entry.setCol(col_posofphrase, Math.min(ie1.posofphrase(),
ie2.posofphrase()));
- ie1.entry.setCol(col_wordsInText, (ie1.wordsintext() +
ie2.wordsintext()) / 2);
+
+ // combine term frequency
+ ie1.entry.setCol(col_wordsInText, ie1.wordsintext() +
ie2.wordsintext());
return ie1;
}
- public void combineDistance(indexRWIEntry oe) {
- combineDistance(this, oe);
+ public void join(indexRWIEntry oe) {
+ join(this, oe);
}
public int worddistance() {
Modified: trunk/source/de/anomic/index/indexRWIVarEntry.java
===================================================================
--- trunk/source/de/anomic/index/indexRWIVarEntry.java 2008-01-31 16:43:34 UTC
(rev 4423)
+++ trunk/source/de/anomic/index/indexRWIVarEntry.java 2008-01-31 23:40:47 UTC
(rev 4424)
@@ -54,7 +54,7 @@
this.phrasesintext = e.phrasesintext();
this.posintext = e.posintext();
this.posinphrase = e.posinphrase();
- this.posofphrase= e.posofphrase();
+ this.posofphrase = e.posofphrase();
this.quality = e.quality();
this.urlcomps = e.urlcomps();
this.urllength = e.urllength();
@@ -65,7 +65,15 @@
this.termFrequency = e.termFrequency();
}
- public void combineDistance(indexRWIEntry oe) {
+ public void join(indexRWIEntry oe) {
+ // combine the distance
+ this.worddistance = this.worddistance() + oe.worddistance() +
Math.abs(this.posintext() - oe.posintext());
+ this.posintext = Math.min(this.posintext(), oe.posintext());
+ this.posinphrase = (this.posofphrase() == oe.posofphrase()) ?
Math.min(this.posinphrase(), oe.posinphrase()) : 0;
+ this.posofphrase = Math.min(this.posofphrase(), oe.posofphrase());
+
+ // combine term frequency
+ this.wordsintext = this.wordsintext() + oe.wordsintext();
}
public kelondroBitfield flags() {
Modified: trunk/source/de/anomic/plasma/plasmaCrawlNURL.java
===================================================================
--- trunk/source/de/anomic/plasma/plasmaCrawlNURL.java 2008-01-31 16:43:34 UTC
(rev 4423)
+++ trunk/source/de/anomic/plasma/plasmaCrawlNURL.java 2008-01-31 23:40:47 UTC
(rev 4424)
@@ -74,10 +74,10 @@
public plasmaCrawlNURL(File cachePath) {
super();
- coreStack = new plasmaCrawlBalancer(cachePath, "urlNoticeCoreStack",
true);
+ coreStack = new plasmaCrawlBalancer(cachePath, "urlNoticeCoreStack",
false);
limitStack = new plasmaCrawlBalancer(cachePath, "urlNoticeLimitStack",
false);
//overhangStack = new plasmaCrawlBalancer(overhangStackFile);
- remoteStack = new plasmaCrawlBalancer(cachePath,
"urlNoticeRemoteStack", true);
+ remoteStack = new plasmaCrawlBalancer(cachePath,
"urlNoticeRemoteStack", false);
}
public void close() {
Modified: trunk/source/de/anomic/plasma/plasmaSearchEvent.java
===================================================================
--- trunk/source/de/anomic/plasma/plasmaSearchEvent.java 2008-01-31
16:43:34 UTC (rev 4423)
+++ trunk/source/de/anomic/plasma/plasmaSearchEvent.java 2008-01-31
23:40:47 UTC (rev 4424)
@@ -103,6 +103,7 @@
this.urlRetrievalAllTime = 0;
this.snippetComputationAllTime = 0;
this.workerThreads = null;
+ this.localSearchThread = null;
this.resultList = new ArrayList<ResultEntry>(10); // this is the
result set which is filled up with search results, enriched with snippets
//this.resultListLock = 0; // no locked elements until now
this.failedURLs = new HashMap<String, String>(); // a map of urls to
reason strings where a worker thread tried to work on, but failed.
@@ -407,6 +408,17 @@
return false;
}
+ private int countFinishedRemoteSearch() {
+ int count = 0;
+ // check only primary search threads
+ if ((this.primarySearchThreads != null) &&
(this.primarySearchThreads.length != 0)) {
+ for (int i = 0; i < this.primarySearchThreads.length; i++) {
+ if ((this.primarySearchThreads[i] == null) ||
(!(this.primarySearchThreads[i].isAlive()))) count++;
+ }
+ }
+ return count;
+ }
+
public plasmaSearchQuery getQuery() {
return query;
}
@@ -563,19 +575,12 @@
public ResultEntry oneResult(int item) {
// first sleep a while to give accumulation threads a chance to work
- if (anyWorkerAlive()) {
- long sleeptime = Math.min(600, this.eventTime +
(this.query.maximumTime / this.query.displayResults() * ((item %
this.query.displayResults()) + 1)) - System.currentTimeMillis());
- if (this.resultList.size() <= item + 10) sleeptime =
Math.min(sleeptime + 300, 600);
- if (sleeptime > 0) try {Thread.sleep(sleeptime);} catch
(InterruptedException e) {}
- //System.out.println("+++DEBUG-oneResult+++ (1) sleeping " +
sleeptime);
-
- // then sleep until any result is available (that should not
happen)
- while ((this.resultList.size() <= item) && (anyWorkerAlive())) {
- try {Thread.sleep(100);} catch (InterruptedException e) {}
- //System.out.println("+++DEBUG-oneResult+++ (2) sleeping " +
100);
- }
-
+ while (((localSearchThread != null) && (localSearchThread.isAlive()))
||
+ ((this.primarySearchThreads != null) &&
(this.primarySearchThreads.length > item) && (anyWorkerAlive()) &&
+ ((this.resultList.size() <= item) ||
(countFinishedRemoteSearch() <= item)))) {
+ try {Thread.sleep(100);} catch (InterruptedException e) {}
}
+
// finally, if there is something, return the result
synchronized (this.resultList) {
// check if we have enough entries
@@ -610,7 +615,7 @@
boolean secondarySearchStartet = false;
private void prepareSecondarySearch() {
- if (secondarySearchStartet) return; // dont do this twice
+ if (secondarySearchStartet) return; // don't do this twice
if ((rcAbstracts == null) || (rcAbstracts.size() !=
query.queryHashes.size())) return; // secondary search not possible (yet)
this.secondarySearchStartet = true;
Modified: trunk/source/de/anomic/plasma/plasmaSwitchboard.java
===================================================================
--- trunk/source/de/anomic/plasma/plasmaSwitchboard.java 2008-01-31
16:43:34 UTC (rev 4423)
+++ trunk/source/de/anomic/plasma/plasmaSwitchboard.java 2008-01-31
23:40:47 UTC (rev 4424)
@@ -877,7 +877,7 @@
private static plasmaSwitchboard sb;
- public plasmaSwitchboard(String rootPath, String initPath, String
configPath, boolean applyPro) {
+ public plasmaSwitchboard(File rootPath, String initPath, String
configPath, boolean applyPro) {
super(rootPath, initPath, configPath, applyPro);
serverProfiling.startSystemProfiling();
sb=this;
Modified: trunk/source/de/anomic/server/logging/serverLog.java
===================================================================
--- trunk/source/de/anomic/server/logging/serverLog.java 2008-01-31
16:43:34 UTC (rev 4423)
+++ trunk/source/de/anomic/server/logging/serverLog.java 2008-01-31
23:40:47 UTC (rev 4424)
@@ -186,7 +186,7 @@
Logger.getLogger(appName).isLoggable(Level.FINEST);
}
- public static final void configureLogging(String homePath, File
loggingConfigFile) throws SecurityException, FileNotFoundException, IOException
{
+ public static final void configureLogging(File homePath, File
loggingConfigFile) throws SecurityException, FileNotFoundException, IOException
{
FileInputStream fileIn = null;
try {
System.out.println("STARTUP: Trying to load logging configuration
from file " + loggingConfigFile.toString());
Modified: trunk/source/de/anomic/server/serverAbstractSwitch.java
===================================================================
--- trunk/source/de/anomic/server/serverAbstractSwitch.java 2008-01-31
16:43:34 UTC (rev 4423)
+++ trunk/source/de/anomic/server/serverAbstractSwitch.java 2008-01-31
23:40:47 UTC (rev 4424)
@@ -58,7 +58,7 @@
// configuration management
private File configFile;
private String configComment;
- private String rootPath;
+ private File rootPath;
protected serverLog log;
protected int serverJobs;
protected long maxTrackingTime;
@@ -69,7 +69,7 @@
private TreeMap<String, serverSwitchAction> switchActions;
protected HashMap<String, TreeMap<Long, String>> accessTracker; //
mappings from requesting host to an ArrayList of serverTrack-entries
- public serverAbstractSwitch(String rootPath, String initPath, String
configPath, boolean applyPro) {
+ public serverAbstractSwitch(File rootPath, String initPath, String
configPath, boolean applyPro) {
// we initialize the switchboard with a property file,
// but maintain these properties then later in a new 'config' file
// to reset all changed configs, the config file must
@@ -526,7 +526,7 @@
public abstract serverObjects action(String actionName, serverObjects
actionInput);
- public String getRootPath() {
+ public File getRootPath() {
return rootPath;
}
Modified: trunk/source/de/anomic/server/serverPlainSwitch.java
===================================================================
--- trunk/source/de/anomic/server/serverPlainSwitch.java 2008-01-31
16:43:34 UTC (rev 4423)
+++ trunk/source/de/anomic/server/serverPlainSwitch.java 2008-01-31
23:40:47 UTC (rev 4424)
@@ -26,9 +26,11 @@
package de.anomic.server;
+import java.io.File;
+
public class serverPlainSwitch extends serverAbstractSwitch implements
serverSwitch {
- public serverPlainSwitch(String rootPath, String initPath, String
configPath, boolean applyPro) {
+ public serverPlainSwitch(File rootPath, String initPath, String
configPath, boolean applyPro) {
super(rootPath, initPath, configPath, applyPro);
}
Modified: trunk/source/de/anomic/server/serverSwitch.java
===================================================================
--- trunk/source/de/anomic/server/serverSwitch.java 2008-01-31 16:43:34 UTC
(rev 4423)
+++ trunk/source/de/anomic/server/serverSwitch.java 2008-01-31 23:40:47 UTC
(rev 4424)
@@ -60,7 +60,7 @@
public interface serverSwitch {
// the root path for the application
- public String getRootPath();
+ public File getRootPath();
// a logger for this switchboard
public void setLog(serverLog log);
Modified: trunk/source/de/anomic/yacy/yacyClient.java
===================================================================
--- trunk/source/de/anomic/yacy/yacyClient.java 2008-01-31 16:43:34 UTC (rev
4423)
+++ trunk/source/de/anomic/yacy/yacyClient.java 2008-01-31 23:40:47 UTC (rev
4424)
@@ -43,6 +43,7 @@
package de.anomic.yacy;
+import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
@@ -1092,7 +1093,7 @@
public static void main(String[] args) {
System.out.println("yacyClient Test");
try {
- final plasmaSwitchboard sb = new plasmaSwitchboard(args[0],
"httpProxy.init", "DATA/SETTINGS/httpProxy.conf", false);
+ final plasmaSwitchboard sb = new plasmaSwitchboard(new
File(args[0]), "httpProxy.init", "DATA/SETTINGS/httpProxy.conf", false);
/*final yacyCore core =*/ new yacyCore(sb);
yacyCore.peerActions.loadSeedLists();
final yacySeed target = yacyCore.seedDB.getConnected(args[1]);
Modified: trunk/source/de/anomic/yacy/yacyVersion.java
===================================================================
--- trunk/source/de/anomic/yacy/yacyVersion.java 2008-01-31 16:43:34 UTC
(rev 4423)
+++ trunk/source/de/anomic/yacy/yacyVersion.java 2008-01-31 23:40:47 UTC
(rev 4424)
@@ -384,6 +384,7 @@
//byte[] script = ("cd " +
plasmaSwitchboard.getSwitchboard().getRootPath() + ";while [ -e ../yacy.running
]; do sleep 1;done;tar xfz " + release + ";cp -Rf yacy/* ../../;rm -Rf yacy;cd
../../;startYACY.sh").getBytes();
try {
plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard();
+ String apphome = sb.getRootPath().toString();
serverLog.logInfo("UPDATE", "INITIATED");
String script =
"#!/bin/sh" + serverCore.LF_STRING +
@@ -394,14 +395,14 @@
"while [ -f ../yacy.running ]; do" + serverCore.LF_STRING +
"sleep 1" + serverCore.LF_STRING +
"done" + serverCore.LF_STRING +
- "cp -Rf yacy/* ../../" + serverCore.LF_STRING +
+ "cp -Rf yacy/* " + apphome + serverCore.LF_STRING +
"rm -Rf yacy" + serverCore.LF_STRING +
"else" + serverCore.LF_STRING +
"while [ -f ../yacy.running ]; do" + serverCore.LF_STRING +
"sleep 1" + serverCore.LF_STRING +
"done" + serverCore.LF_STRING +
"fi" + serverCore.LF_STRING +
- "cd ../../" + serverCore.LF_STRING +
+ "cd " + apphome + serverCore.LF_STRING +
"nohup ./startYACY.sh > /dev/null" + serverCore.LF_STRING;
File scriptFile = new File(sb.getRootPath(),
"DATA/RELEASE/update.sh");
serverSystem.deployScript(scriptFile, script);
Modified: trunk/source/migration.java
===================================================================
--- trunk/source/migration.java 2008-01-31 16:43:34 UTC (rev 4423)
+++ trunk/source/migration.java 2008-01-31 23:40:47 UTC (rev 4424)
@@ -51,7 +51,7 @@
//SVN constants
public static final int USE_WORK_DIR=1389; //wiki & messages in DATA/WORK
public static final int TAGDB_WITH_TAGHASH=1635; //tagDB keys are
tagHashes instead of plain tagname.
- public static final int NEW_OVERLAYS=3675;
+ public static final int NEW_OVERLAYS=4422;
public static void main(String[] args) {
}
Modified: trunk/source/yacy.java
===================================================================
--- trunk/source/yacy.java 2008-01-31 16:43:34 UTC (rev 4423)
+++ trunk/source/yacy.java 2008-01-31 23:40:47 UTC (rev 4424)
@@ -164,7 +164,7 @@
* @param homePath Root-path where all information is to be found.
* @param startupFree free memory at startup time, to be used later for
statistics
*/
- private static void startup(String homePath, long startupMemFree, long
startupMemTotal) {
+ private static void startup(File homePath, long startupMemFree, long
startupMemTotal) {
int oldRev=0;
int newRev=0;
@@ -183,7 +183,7 @@
}
// ensure that there is a DATA directory, if not, create one and
if that fails warn and die
- File f = new File(homePath); if (!(f.exists())) f.mkdirs();
+ File f = homePath; if (!(f.exists())) f.mkdirs();
f = new File(homePath, "DATA/"); if (!(f.exists())) f.mkdirs();
if (!(f.exists())) {
System.err.println("Error creating
DATA-directory in " + homePath.toString() + " . Please check your
write-permission for this folder. YaCy will now terminate.");
@@ -198,7 +198,7 @@
System.out.println("could not copy yacy.logging");
}
try{
- serverLog.configureLogging(homePath,new File(homePath,
"DATA/LOG/yacy.logging"));
+ serverLog.configureLogging(homePath, new File(homePath,
"DATA/LOG/yacy.logging"));
} catch (IOException e) {
System.out.println("could not find logging properties in
homePath=" + homePath);
e.printStackTrace();
@@ -263,7 +263,7 @@
sb.setConfig("version", Double.toString(version));
sb.setConfig("vString",
yacyVersion.combined2prettyVersion(Double.toString(version)));
sb.setConfig("vdate", (vDATE.startsWith("@")) ?
serverDate.formatShortDay() : vDATE);
- sb.setConfig("applicationRoot", homePath);
+ sb.setConfig("applicationRoot", homePath.toString());
serverLog.logConfig("STARTUP", "YACY Version: " + version + ",
Built " + sb.getConfig("vdate", "00000000"));
yacyVersion.latestRelease = version;
@@ -451,7 +451,7 @@
* @param homePath Root-path where all the information is to be found.
* @return Properties read from the configurationfile.
*/
- private static Properties configuration(String mes, String homePath) {
+ private static Properties configuration(String mes, File homePath) {
serverLog.logConfig(mes, "Application Root Path: " +
homePath.toString());
// read data folder
@@ -480,7 +480,7 @@
// YaCy is running in the same runtime. we can shutdown via
interrupt
sb.terminate();
} else {
- String applicationRoot =
System.getProperty("user.dir").replace('\\', '/');
+ File applicationRoot = new
File(System.getProperty("user.dir").replace('\\', '/'));
shutdown(applicationRoot);
}
}
@@ -491,7 +491,7 @@
*
* @param homePath Root-path where all the information is to be found.
*/
- static void shutdown(String homePath) {
+ static void shutdown(File homePath) {
// start up
System.out.println(copyright);
System.out.println(hline);
@@ -544,7 +544,7 @@
*
* @param homePath Root-Path where all the information is to be found.
*/
- private static void genWordstat(String homePath) {
+ private static void genWordstat(File homePath) {
// start up
System.out.println(copyright);
System.out.println(hline);
@@ -587,12 +587,12 @@
* @param homePath path to the YaCy directory
* @param dbcache cache size in MB
*/
- public static void minimizeUrlDB(String homePath) {
+ public static void minimizeUrlDB(File homePath) {
// run with "java -classpath classes yacy -minimizeUrlDB"
- try {serverLog.configureLogging(homePath,new File(homePath,
"DATA/LOG/yacy.logging"));} catch (Exception e) {}
- File indexPrimaryRoot = new File(new File(homePath), "DATA/INDEX");
- File indexSecondaryRoot = new File(new File(homePath), "DATA/INDEX");
- File indexRoot2 = new File(new File(homePath), "DATA/INDEX2");
+ try {serverLog.configureLogging(homePath, new File(homePath,
"DATA/LOG/yacy.logging"));} catch (Exception e) {}
+ File indexPrimaryRoot = new File(homePath, "DATA/INDEX");
+ File indexSecondaryRoot = new File(homePath, "DATA/INDEX");
+ File indexRoot2 = new File(homePath, "DATA/INDEX2");
serverLog log = new serverLog("URL-CLEANUP");
try {
log.logInfo("STARTING URL CLEANUP");
@@ -777,24 +777,24 @@
*
* @param homePath Root-Path where all information is to be found.
*/
- private static void urldbcleanup(String homePath) {
- File root = new File(homePath);
+ private static void urldbcleanup(File homePath) {
+ File root = homePath;
File indexroot = new File(root, "DATA/INDEX");
- try {serverLog.configureLogging(homePath,new File(homePath,
"DATA/LOG/yacy.logging"));} catch (Exception e) {}
+ try {serverLog.configureLogging(homePath, new File(homePath,
"DATA/LOG/yacy.logging"));} catch (Exception e) {}
plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexroot, 10000);
currentUrlDB.urldbcleanup();
currentUrlDB.close();
}
- private static void RWIHashList(String homePath, String targetName, String
resource, String format) {
+ private static void RWIHashList(File homePath, String targetName, String
resource, String format) {
plasmaWordIndex WordIndex = null;
serverLog log = new serverLog("HASHLIST");
- File indexPrimaryRoot = new File(new File(homePath), "DATA/INDEX");
- File indexSecondaryRoot = new File(new File(homePath), "DATA/INDEX");
+ File indexPrimaryRoot = new File(homePath, "DATA/INDEX");
+ File indexSecondaryRoot = new File(homePath, "DATA/INDEX");
String wordChunkStartHash = "AAAAAAAAAAAA";
- try {serverLog.configureLogging(homePath,new File(homePath,
"DATA/LOG/yacy.logging"));} catch (Exception e) {}
+ try {serverLog.configureLogging(homePath, new File(homePath,
"DATA/LOG/yacy.logging"));} catch (Exception e) {}
log.logInfo("STARTING CREATION OF RWI-HASHLIST");
- File root = new File(homePath);
+ File root = homePath;
try {
Iterator<indexContainer> indexContainerIterator = null;
if (resource.equals("all")) {
@@ -850,10 +850,10 @@
* Searching for peers affected by Bug
* @param homePath
*/
- public static void testPeerDB(String homePath) {
+ public static void testPeerDB(File homePath) {
try {
- File yacyDBPath = new File(new File(homePath), "DATA/YACYDB");
+ File yacyDBPath = new File(homePath, "DATA/YACYDB");
String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) {
@@ -905,16 +905,16 @@
// go into headless awt mode
System.setProperty("java.awt.headless", "true");
- String applicationRoot = System.getProperty("user.dir").replace('\\',
'/');
+ File applicationRoot = new
File(System.getProperty("user.dir").replace('\\', '/'));
//System.out.println("args.length=" + args.length);
//System.out.print("args=["); for (int i = 0; i < args.length; i++)
System.out.print(args[i] + ", "); System.out.println("]");
if ((args.length >= 1) && ((args[0].toLowerCase().equals("-startup"))
|| (args[0].equals("-start")))) {
// normal start-up of yacy
- if (args.length == 2) applicationRoot= args[1];
+ if (args.length == 2) applicationRoot= new File(args[1]);
startup(applicationRoot, startupMemFree, startupMemTotal);
} else if ((args.length >= 1) &&
((args[0].toLowerCase().equals("-shutdown")) || (args[0].equals("-stop")))) {
// normal shutdown of yacy
- if (args.length == 2) applicationRoot= args[1];
+ if (args.length == 2) applicationRoot= new File(args[1]);
shutdown(applicationRoot);
} else if ((args.length >= 1) &&
(args[0].toLowerCase().equals("-minimizeurldb"))) {
// migrate words from DATA/PLASMADB/WORDS path to assortment
cache, if possible
@@ -922,11 +922,11 @@
if (args.length >= 3 && args[1].toLowerCase().equals("-cache")) {
args = shift(args, 1, 2);
}
- if (args.length == 2) applicationRoot= args[1];
+ if (args.length == 2) applicationRoot= new File(args[1]);
minimizeUrlDB(applicationRoot);
} else if ((args.length >= 1) &&
(args[0].toLowerCase().equals("-testpeerdb"))) {
if (args.length == 2) {
- applicationRoot= args[1];
+ applicationRoot = new File(args[1]);
} else if (args.length > 2) {
System.err.println("Usage: -testPeerDB [homeDbRoot]");
}
@@ -935,7 +935,7 @@
// this can help to create a stop-word list
// to use this, you need a 'yacy.words' file in the root path
// start this with "java -classpath classes yacy -genwordstat
[<rootdir>]"
- if (args.length == 2) applicationRoot= args[1];
+ if (args.length == 2) applicationRoot= new File(args[1]);
genWordstat(applicationRoot);
} else if ((args.length == 4) &&
(args[0].toLowerCase().equals("-cleanwordlist"))) {
// this can be used to organize and clean a word-list
@@ -950,7 +950,7 @@
transferCR(targetaddress, crfile);
} else if ((args.length >= 1) &&
(args[0].toLowerCase().equals("-urldbcleanup"))) {
// generate a url list and save it in a file
- if (args.length == 2) applicationRoot= args[1];
+ if (args.length == 2) applicationRoot= new File(args[1]);
urldbcleanup(applicationRoot);
} else if ((args.length >= 1) &&
(args[0].toLowerCase().equals("-rwihashlist"))) {
// generate a url list and save it in a file
@@ -958,11 +958,11 @@
String format = "txt";
if (args.length >= 2) domain= args[1];
if (args.length >= 3) format= args[2];
- if (args.length == 4) applicationRoot= args[3];
+ if (args.length == 4) applicationRoot= new File(args[3]);
String outfile = "rwihashlist_" + System.currentTimeMillis();
RWIHashList(applicationRoot, outfile, domain, format);
} else {
- if (args.length == 1) applicationRoot= args[0];
+ if (args.length == 1) applicationRoot= new File(args[0]);
startup(applicationRoot, startupMemFree, startupMemTotal);
}
}
_______________________________________________
YaCy-svn mailing list
YaCy-svn@lists.berlios.de
https://lists.berlios.de/mailman/listinfo/yacy-svn
