Author: borg-0300 Date: 2008-02-10 16:57:52 +0100 (Sun, 10 Feb 2008) New Revision: 4472
Modified: trunk/source/de/anomic/yacy/yacyCore.java trunk/source/de/anomic/yacy/yacySeed.java trunk/source/de/anomic/yacy/yacySeedDB.java trunk/yacy.init Log: better ping Modified: trunk/source/de/anomic/yacy/yacyCore.java =================================================================== --- trunk/source/de/anomic/yacy/yacyCore.java 2008-02-10 09:06:17 UTC (rev 4471) +++ trunk/source/de/anomic/yacy/yacyCore.java 2008-02-10 15:57:52 UTC (rev 4472) @@ -81,24 +81,23 @@ // statics public static final ThreadGroup publishThreadGroup = new ThreadGroup("publishThreadGroup"); - public static yacySeedDB seedDB = null; - public static yacyNewsPool newsPool = null; - public static final HashMap<String, String> seedUploadMethods = new HashMap<String, String>(); - public static yacyPeerActions peerActions = null; - public static yacyDHTAction dhtAgent = null; + public static yacySeedDB seedDB; + public static yacyNewsPool newsPool; + public static final HashMap seedUploadMethods = new HashMap(); + public static yacyPeerActions peerActions; + public static yacyDHTAction dhtAgent; public static serverLog log; - public static long lastOnlineTime = 0; - /** pseudo-random key derived from a time-interval while YaCy startup*/ - public static long speedKey = 0; + public static long lastOnlineTime; + /** pseudo-random key derived from a time-interval while YaCy startup. */ + public static long speedKey; public static File yacyDBPath; - public static final Map<String, yacyAccessible> amIAccessibleDB = Collections.synchronizedMap(new HashMap<String, yacyAccessible>()); // Holds PeerHash / yacyAccessible Relations + public static final Map amIAccessibleDB = Collections.synchronizedMap(new HashMap()); // Holds PeerHash / yacyAccessible Relations // constants for PeerPing behaviour - private static final int PING_INITIAL = 16; - private static final int PING_MAX_RUNNING = 8; - private static final int PING_MIN_RUNNING = 4; + private static final int PING_INITIAL = 64; + private static final int PING_MIN_LASTSEEN = 240000; // in milliseconds private static final int PING_MIN_PEERSEEN = 2; // min. accessible to force senior private static final long PING_MAX_DBAGE = 15 * 60 * 1000; // in milliseconds - + // public static yacyShare shareManager = null; // public static boolean terminate = false; @@ -372,8 +371,8 @@ int attempts = seedDB.sizeConnected(); // getting a list of peers to contact - if (seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) { - if (attempts > PING_INITIAL) { attempts = PING_INITIAL; } + if (seedDB.mySeed().isVirgin()) { + attempts = Math.min(attempts, PING_INITIAL); final Map<String, String> ch = plasmaSwitchboard.getSwitchboard().clusterhashes; seeds = seedDB.seedsByAge(true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection // add also all peers from cluster if this is a public robinson cluster @@ -395,21 +394,11 @@ } } } else { - if (amIAccessibleDB.size() > PING_MAX_RUNNING) { - attempts = PING_MAX_RUNNING; - } else { - attempts = Math.min(attempts, PING_MIN_RUNNING); - } - seeds = seedDB.seedsByAge(false, attempts); // best for seed list maintenance/cleaning + seeds = seedDB.getOldestSeeds(attempts / 5, PING_MIN_LASTSEEN); } - if (seeds == null || seeds.size() == 0) { return 0; } - if (seeds.size() < attempts) { attempts = seeds.size(); } + attempts = seeds.size(); - // This will try to get Peers that are not currently in amIAccessibleDB - Iterator<yacySeed> si = seeds.values().iterator(); - yacySeed seed; - // include a YaCyNews record to my seed try { final yacyNewsRecord record = newsPool.myPublication(); @@ -433,10 +422,14 @@ final List<Thread> syncList = Collections.synchronizedList(new LinkedList<Thread>()); // memory for threads final serverSemaphore sync = new serverSemaphore(attempts); + // This will try to get Peers that are not currently in amIAccessibleDB + Iterator<yacySeed> si = seeds.values().iterator(); + yacySeed seed; + // going through the peer list and starting a new publisher thread for each peer int i = 0; while (si.hasNext()) { - seed = (yacySeed) si.next(); + seed = si.next(); if (seed == null) { sync.P(); continue; @@ -445,8 +438,8 @@ final String address = seed.getClusterAddress(); log.logFine("HELLO #" + i + " to peer '" + seed.get(yacySeed.NAME, "") + "' at " + address); // debug - String seederror = seed.isProper(); - if ((address == null) || (seederror != null)) { + final String seederror = seed.isProper(); + if (address == null || seederror != null) { // we don't like that address, delete it peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " + address + "; seederror = " + seederror); sync.P(); @@ -489,9 +482,9 @@ final int dbSize; synchronized (amIAccessibleDB) { dbSize = amIAccessibleDB.size(); - Iterator<String> ai = amIAccessibleDB.keySet().iterator(); + final Iterator<String> ai = amIAccessibleDB.keySet().iterator(); while (ai.hasNext()) { - yacyAccessible ya = (yacyAccessible) amIAccessibleDB.get(ai.next()); + final yacyAccessible ya = (yacyAccessible) amIAccessibleDB.get(ai.next()); if (ya.lastUpdated < cutofftime) { ai.remove(); } else { @@ -507,11 +500,11 @@ log.logInfo("PeerPing: I am accessible for " + accessible + " peer(s), not accessible for " + notaccessible + " peer(s)."); - if ((accessible + notaccessible) > 0) { + if (accessible + notaccessible > 0) { final String newPeerType; // At least one other Peer told us our type - if ((accessible >= PING_MIN_PEERSEEN) || - (accessible >= notaccessible)) { + if (accessible >= PING_MIN_PEERSEEN || + accessible >= notaccessible) { // We can be reached from a majority of other Peers if (yacyCore.seedDB.mySeed().isPrincipal()) { newPeerType = yacySeed.PEERTYPE_PRINCIPAL; @@ -522,11 +515,11 @@ // We cannot be reached from the outside newPeerType = yacySeed.PEERTYPE_JUNIOR; } - if (yacyCore.seedDB.mySeed().orVirgin().equals(newPeerType)) { + if (yacyCore.seedDB.mySeed().isType(newPeerType)) { log.logInfo("PeerPing: myType is " + yacyCore.seedDB.mySeed().orVirgin()); } else { log.logInfo("PeerPing: changing myType from '" + yacyCore.seedDB.mySeed().orVirgin() + "' to '" + newPeerType + "'"); - yacyCore.seedDB.mySeed().put(yacySeed.PEERTYPE, newPeerType); + yacyCore.seedDB.mySeed().setType(newPeerType); } } else { log.logInfo("PeerPing: No data, staying at myType: " + yacyCore.seedDB.mySeed().orVirgin()); Modified: trunk/source/de/anomic/yacy/yacySeed.java =================================================================== --- trunk/source/de/anomic/yacy/yacySeed.java 2008-02-10 09:06:17 UTC (rev 4471) +++ trunk/source/de/anomic/yacy/yacySeed.java 2008-02-10 15:57:52 UTC (rev 4472) @@ -84,7 +84,7 @@ public class yacySeed { - public static final int maxsize = 4096; + public static final int maxsize = 4096; /** * <b>substance</b> "sI" (send index/words) */ @@ -281,9 +281,9 @@ * that is used instead the address stored in the seed DNA */ public void setAlternativeAddress(String ipport) { - if (ipport == null) return; - int p = ipport.indexOf(':'); - if (p < 0) this.alternativeIP = ipport; else this.alternativeIP = ipport.substring(0, p); + if (ipport == null) return; + int p = ipport.indexOf(':'); + if (p < 0) this.alternativeIP = ipport; else this.alternativeIP = ipport.substring(0, p); } /** @@ -298,7 +298,7 @@ public final String getPeerType() { return get(yacySeed.PEERTYPE, ""); } /** * try to get the peertype<br> - * @return the peertype or "Virgin" + * @return the peertype or "virgin" */ public final String orVirgin() { return get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN); } /** @@ -342,14 +342,14 @@ } else return dflt; } - public final void setIP() { dna.put(yacySeed.IP, ""); } - public final void setIP(final String ip) { dna.put(yacySeed.IP, ip); } - public final void setPort(final String port) { dna.put(yacySeed.PORT, port); } - public final void setJunior() { dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); } - public final void setSenior() { dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); } - public final void setPrincipal() { dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL); } - - + public final void setIP() { dna.put(yacySeed.IP, ""); } + public final void setIP(String ip) { dna.put(yacySeed.IP, ip); } + public final void setPort(String port) { dna.put(yacySeed.PORT, port); } + public final void setType(String type) { dna.put(yacySeed.PEERTYPE, type); } + public final void setJunior() { dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR); } + public final void setSenior() { dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); } + public final void setPrincipal() { dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL); } + public final void put(String key, String value) { synchronized (this.dna) { this.dna.put(key, value); @@ -643,8 +643,8 @@ public final void setUnusedFlags() { for (int i = 4; i < 24; i++) { setFlag(i, true); } } - public final boolean isPotential() { - return isVirgin() || isJunior(); + public final boolean isType(String type) { + return get(yacySeed.PEERTYPE, "").equals(type); } public final boolean isVirgin() { return get(yacySeed.PEERTYPE, "").equals(yacySeed.PEERTYPE_VIRGIN); @@ -652,22 +652,25 @@ public final boolean isJunior() { return get(yacySeed.PEERTYPE, "").equals(yacySeed.PEERTYPE_JUNIOR); } - public final boolean isActive() { - return isSenior() || isPrincipal(); - } public final boolean isSenior() { return get(yacySeed.PEERTYPE, "").equals(yacySeed.PEERTYPE_SENIOR); } public final boolean isPrincipal() { return get(yacySeed.PEERTYPE, "").equals(yacySeed.PEERTYPE_PRINCIPAL); } + public final boolean isPotential() { + return isVirgin() || isJunior(); + } + public final boolean isActive() { + return isSenior() || isPrincipal(); + } public final boolean isOnline() { return isSenior() || isPrincipal(); } public final boolean isOnline(final String type) { return type.equals(yacySeed.PEERTYPE_SENIOR) || type.equals(yacySeed.PEERTYPE_PRINCIPAL); } - + public static final long minDHTNumber = kelondroBase64Order.enhancedCoder.cardinal(kelondroBase64Order.zero(12)); public static final long maxDHTDistance = Long.MAX_VALUE; Modified: trunk/source/de/anomic/yacy/yacySeedDB.java =================================================================== --- trunk/source/de/anomic/yacy/yacySeedDB.java 2008-02-10 09:06:17 UTC (rev 4471) +++ trunk/source/de/anomic/yacy/yacySeedDB.java 2008-02-10 15:57:52 UTC (rev 4472) @@ -381,6 +381,40 @@ } } + public HashMap<String, yacySeed> getOldestSeeds(int count, int minage) { + // returns a peerhash/yacySeed relation + try { + final kelondroMScoreCluster seedScore = new kelondroMScoreCluster(); + final Iterator<yacySeed> s = seedsConnected(true, false, null, (float) 0.0); + yacySeed ys; + int age; + int searchcount = 1000; + while (s.hasNext() && searchcount-- > 0) { + ys = s.next(); + if (ys == null) { continue; } + age = (int) Math.abs(System.currentTimeMillis() + serverDate.dayMillis - ys.getLastSeenUTC()); + if (age < minage) { continue; } + seedScore.addScore(ys.hash, age); // the higher age, the older is the peer + } + if (seedScore.size() == 0) { return null; } + + // result is now in the score object; create a result vector + final HashMap<String, yacySeed> result = new HashMap<String, yacySeed>(); + final Iterator<String> it = seedScore.scores(false); + searchcount = Math.min(count, seedScore.size()); + int c = 0; + while (c++ < searchcount && it.hasNext()) { + ys = getConnected(it.next()); + if (ys != null && ys.hash != null) { result.put(ys.hash, ys); } + } + return result; + } catch (kelondroException e) { + seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile); + yacyCore.log.logFine("Internal Error at yacySeedDB.seedsByAge: " + e.getMessage(), e); + return null; + } + } + public int sizeConnected() { return seedActiveDB.size(); /* Modified: trunk/yacy.init =================================================================== --- trunk/yacy.init 2008-02-10 09:06:17 UTC (rev 4471) +++ trunk/yacy.init 2008-02-10 15:57:52 UTC (rev 4472) @@ -546,8 +546,8 @@ 20_dhtdistribution_idlesleep=30000 20_dhtdistribution_busysleep=10000 20_dhtdistribution_memprereq=6291456 -30_peerping_idlesleep=90000 -30_peerping_busysleep=90000 +30_peerping_idlesleep=120000 +30_peerping_busysleep=120000 30_peerping_memprereq=1048576 40_peerseedcycle_idlesleep=1800000 40_peerseedcycle_busysleep=1200000 _______________________________________________ YaCy-svn mailing list YaCy-svn@lists.berlios.de https://lists.berlios.de/mailman/listinfo/yacy-svn