Author: orbiter
Date: 2008-03-10 00:58:22 +0100 (Mon, 10 Mar 2008)
New Revision: 4543
Modified:
trunk/htroot/yacy/user/sidebar_navigation.java
trunk/source/de/anomic/http/httpdFileHandler.java
trunk/source/de/anomic/kelondro/kelondroCollectionIndex.java
trunk/source/de/anomic/plasma/plasmaSearchRankingProcess.java
trunk/source/de/anomic/yacy/yacyURL.java
Log:
- one try to fix the httpd problem
- fix for handling of collection index that appears when removing elements
- added another navigation method (stub, not working yet)
Modified: trunk/htroot/yacy/user/sidebar_navigation.java
===================================================================
--- trunk/htroot/yacy/user/sidebar_navigation.java 2008-03-09 23:48:24 UTC
(rev 4542)
+++ trunk/htroot/yacy/user/sidebar_navigation.java 2008-03-09 23:58:22 UTC
(rev 4543)
@@ -25,6 +25,7 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.util.Iterator;
+import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
@@ -36,6 +37,7 @@
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
+import de.anomic.yacy.yacyURL;
public class sidebar_navigation {
@@ -119,15 +121,34 @@
prop.put("navigation_topwords_words_" + hintcount +
"_resource", theQuery.searchdom());
prop.put("navigation_topwords_words_" + hintcount +
"_zonecode", theQuery.zonecode);
}
- prop.put("navigation_topwords_words", hintcount);
- if (hintcount++ > MAX_TOPWORDS) {
- break;
- }
+ hintcount++;
+ if (hintcount >= MAX_TOPWORDS) break;
}
+ prop.put("navigation_topwords_words", hintcount);
prop.put("navigation_topwords", "1");
}
}
+ // compose language zone drill-down
+ int c = 0;
+ final Iterator<Map.Entry<String, Integer>> iter =
theSearch.getRankingResult().getZoneStatistics().entrySet().iterator();
+ Map.Entry<String, Integer> entry;
+ while (iter.hasNext()) {
+ entry = iter.next();
+ if ((theQuery == null) || (theQuery.queryString == null)) break;
+ prop.putHTML("navigation_languagezone_zones_" + c + "_zone",
entry.getKey() + " (" + entry.getValue() + ")");
+ prop.putHTML("navigation_languagezone_zones_" + c + "_search",
theQuery.queryString.replace(' ', '+'));
+ prop.put("navigation_languagezone_zones_" + c + "_count",
theQuery.displayResults());
+ prop.put("navigation_languagezone_zones_" + c + "_offset", "0");
+ prop.put("navigation_languagezone_zones_" + c + "_contentdom",
theQuery.contentdom());
+ prop.put("navigation_languagezone_zones_" + c + "_resource",
theQuery.searchdom());
+ prop.put("navigation_languagezone_zones_" + c + "_zonecode",
yacyURL.zone2map.get(entry.getKey()).intValue());
+ prop.put("navigation_languagezone_zones", c);
+ c++;
+ }
+ prop.put("navigation_languagezone", (c > 2) ? "1" : "0");
+
+
// compose page navigation
StringBuffer resnav = new StringBuffer();
int thispage = offset / theQuery.displayResults();
Modified: trunk/source/de/anomic/http/httpdFileHandler.java
===================================================================
--- trunk/source/de/anomic/http/httpdFileHandler.java 2008-03-09 23:48:24 UTC
(rev 4542)
+++ trunk/source/de/anomic/http/httpdFileHandler.java 2008-03-09 23:58:22 UTC
(rev 4543)
@@ -851,7 +851,17 @@
if (chunkedOut != null) {
chunkedOut.finish();
}
- }
+
+ // flush all
+ try {newOut.flush();}catch (Exception e) {}
+
+ // wait a little time until everything closes so that
clients can read from the streams/sockets
+ if ((contentLength >= 0) &&
((String)requestHeader.get(httpHeader.CONNECTION,
"close")).indexOf("keep-alive") == -1) {
+ // in case that the client knows the size in
advance (contentLength present) the waiting will have no effect on the
interface performance
+ // but if the client waits on a connection
interruption this will slow down.
+ try {Thread.sleep(2000);} catch
(InterruptedException e) {} // FIXME: is this necessary?
+ }
+ }
// check mime type again using the result array: these are
'magics'
// if (serverByteBuffer.equals(result, 1,
"PNG".getBytes())) mimeType = mimeTable.getProperty("png","text/html");
@@ -921,10 +931,6 @@
} finally {
try {out.flush();}catch (Exception e) {}
- if (((String)requestHeader.get(httpHeader.CONNECTION,
"close")).indexOf("keep-alive") == -1) {
- // wait a little time until everything closes so that clients
can read from the streams/sockets
- try {Thread.sleep(50);} catch (InterruptedException e) {} //
FIXME: is this necessary?
- }
}
}
Modified: trunk/source/de/anomic/kelondro/kelondroCollectionIndex.java
===================================================================
--- trunk/source/de/anomic/kelondro/kelondroCollectionIndex.java
2008-03-09 23:48:24 UTC (rev 4542)
+++ trunk/source/de/anomic/kelondro/kelondroCollectionIndex.java
2008-03-09 23:58:22 UTC (rev 4543)
@@ -920,7 +920,7 @@
array_remove(
oldPartitionNumber, serialNumber,
this.payloadrow.objectsize,
oldrownumber);
- index.remove(key, true);
+ index.remove(key, false);
return removed;
}
Modified: trunk/source/de/anomic/plasma/plasmaSearchRankingProcess.java
===================================================================
--- trunk/source/de/anomic/plasma/plasmaSearchRankingProcess.java
2008-03-09 23:48:24 UTC (rev 4542)
+++ trunk/source/de/anomic/plasma/plasmaSearchRankingProcess.java
2008-03-09 23:58:22 UTC (rev 4543)
@@ -68,6 +68,7 @@
private TreeSet<String> misses; // contains url-hashes that could not been
found in the LURL-DB
private plasmaWordIndex wordIndex;
private HashMap<String, indexContainer>[] localSearchContainerMaps;
+ private int[] domZones;
public plasmaSearchRankingProcess(plasmaWordIndex wordIndex,
plasmaSearchQuery query, int maxentries, int concurrency) {
// we collect the urlhashes and construct a list with urlEntry objects
@@ -90,6 +91,8 @@
this.wordIndex = wordIndex;
this.flagcount = new int[32];
for (int i = 0; i < 32; i++) {this.flagcount[i] = 0;}
+ this.domZones = new int[8];
+ for (int i = 0; i < 8; i++) {this.domZones[i] = 0;}
}
public long ranking(indexRWIVarEntry word) {
@@ -176,6 +179,10 @@
continue;
}
+ // count domZones
+ System.out.println("DEBUG domDomain dom=" +
wordIndex.loadedURL.load(iEntry.urlHash, iEntry, 0).comp().url().getHost() + ",
zone=" + yacyURL.domDomain(iEntry.urlHash()));
+ this.domZones[yacyURL.domDomain(iEntry.urlHash())]++;
+
// insert
if ((maxentries < 0) || (stack.size() < maxentries)) {
// in case that we don't have enough yet, accept any new entry
@@ -327,6 +334,10 @@
return this.local_resourceSize;
}
+ public Map<String, Integer> getZoneStatistics() {
+ return yacyURL.zoneStatistics(this.domZones);
+ }
+
public indexRWIEntry remove(String urlHash) {
kelondroSortStack<indexRWIVarEntry>.stackElement se =
stack.remove(urlHash.hashCode());
if (se == null) return null;
Modified: trunk/source/de/anomic/yacy/yacyURL.java
===================================================================
--- trunk/source/de/anomic/yacy/yacyURL.java 2008-03-09 23:48:24 UTC (rev
4542)
+++ trunk/source/de/anomic/yacy/yacyURL.java 2008-03-09 23:58:22 UTC (rev
4543)
@@ -31,6 +31,7 @@
import java.net.MalformedURLException;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.Map;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -382,7 +383,7 @@
public static final int language_domain_africa_zone = 128 + 32;
//{5, 7};
public static final int language_domain_any_zone = 255;
- public static final String[] regions = {"europe", "english", "spanish",
"asia", "middleeast", "africa"};
+ public static final HashMap<String, Integer> zone2map = new
HashMap<String, Integer>();
static {
// create a dummy hash
@@ -398,6 +399,14 @@
insertTLDProps(TLD_Africa, 5); // africa
insertTLDProps(TLD_Generic, 6); // anything else, mixed
languages, mainly english
// the id=7 is used to flag local addresses
+
+ zone2map.put("europe", language_domain_europe_zone);
+ zone2map.put("english", language_domain_english_zone);
+ zone2map.put("spanish", language_domain_spanish_zone);
+ zone2map.put("asia", language_domain_asia_zone);
+ zone2map.put("middleeast", language_domain_middleeast_zone);
+ zone2map.put("africa", language_domain_africa_zone);
+ zone2map.put("any", language_domain_any_zone);
}
// class variables
@@ -1093,7 +1102,7 @@
// returns the ID of the domain of the domain
assert (urlHash != null);
assert (urlHash.length() == 12) : "urlhash = " + urlHash;
- return
(kelondroBase64Order.enhancedCoder.decodeByte(urlHash.charAt(11)) & 12) >> 2;
+ return
(kelondroBase64Order.enhancedCoder.decodeByte(urlHash.charAt(11)) & 28) >> 2;
}
public static boolean isLocalDomain(String urlhash) {
@@ -1123,6 +1132,26 @@
return language;
}
+ public static Map<String, Integer> zoneStatistics(int[] domAccumulators) {
+ assert domAccumulators.length == 8;
+ HashMap<String, Integer> zoneCounter = new HashMap<String, Integer>();
+ Iterator<Map.Entry<String, Integer>> j;
+ Map.Entry<String, Integer> entry;
+ for (int i = 0; i < 8; i++) {
+ j = zone2map.entrySet().iterator();
+ while (j.hasNext()) {
+ entry = j.next();
+ if ((i & entry.getValue().intValue()) != 0) {
+ if (zoneCounter.containsKey(entry.getKey())) {
+ zoneCounter.put(entry.getKey(),
zoneCounter.get(entry.getKey()) + domAccumulators[i]);
+ } else {
+ zoneCounter.put(entry.getKey(), domAccumulators[i]);
+ }
+ }
+ }
+ }
+ return zoneCounter;
+ }
public static void main(String[] args) {
String[][] test = new String[][]{
_______________________________________________
YaCy-svn mailing list
[email protected]
https://lists.berlios.de/mailman/listinfo/yacy-svn
