Author: orbiter
Date: 2008-01-19 02:50:24 +0100 (Sat, 19 Jan 2008)
New Revision: 4345
Modified:
trunk/source/de/anomic/kelondro/kelondroCollectionIndex.java
trunk/source/de/anomic/kelondro/kelondroEcoTable.java
trunk/source/de/anomic/plasma/plasmaCrawlBalancer.java
trunk/source/de/anomic/plasma/plasmaCrawlNURL.java
Log:
better RAM protection using eco tables
Modified: trunk/source/de/anomic/kelondro/kelondroCollectionIndex.java
===================================================================
--- trunk/source/de/anomic/kelondro/kelondroCollectionIndex.java
2008-01-19 01:07:52 UTC (rev 4344)
+++ trunk/source/de/anomic/kelondro/kelondroCollectionIndex.java
2008-01-19 01:50:24 UTC (rev 4345)
@@ -154,12 +154,17 @@
}
}
serverLog.logFine("STARTUP", "STARTED INITIALIZATION OF NEW
COLLECTION INDEX WITH " + initialSpace + " ENTRIES. THIS WILL TAKE SOME TIME");
-
+ kelondroRow indexRowdef = indexRow(keyLength, indexOrder);
+ long necessaryRAM4fullTable = minimumRAM4Eco +
(indexRowdef.objectsize + 4) * initialSpace * 3 / 2;
+ long necessaryRAM4fullIndex = minimumRAM4Eco +
(indexRowdef.primaryKeyLength + 4) * initialSpace * 3 / 2;
+
// initialize (new generation) index table from file
- if (serverMemory.request(minimumRAM4Eco, false)) {
- index = new kelondroEcoTable(f, indexRow(keyLength,
indexOrder), true, EcoFSBufferSize);
+ if (serverMemory.request(necessaryRAM4fullTable, false)) {
+ index = new kelondroEcoTable(f, indexRowdef, true,
EcoFSBufferSize);
+ } else if (serverMemory.request(necessaryRAM4fullIndex, false)) {
+ index = new kelondroEcoTable(f, indexRowdef, false,
EcoFSBufferSize);
} else {
- index = new kelondroFlexTable(path, filenameStub + ".index",
preloadTime, indexRow(keyLength, indexOrder), initialSpace, true);
+ index = new kelondroFlexTable(path, filenameStub + ".index",
preloadTime, indexRowdef, initialSpace, true);
}
// open array files
@@ -233,9 +238,11 @@
long preloadTime, int loadfactor, kelondroRow rowdef, int
initialSpace) throws IOException {
// open/create index table
File f = new File(path, filenameStub + ".index");
+ kelondroRow indexRowdef = indexRow(keylength, indexOrder);
+
if (f.isDirectory()) {
// use a flextable
- kelondroIndex theindex = new kelondroCache(new
kelondroFlexTable(path, filenameStub + ".index", preloadTime,
indexRow(keylength, indexOrder), initialSpace, true));
+ kelondroIndex theindex = new kelondroCache(new
kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRowdef,
initialSpace, true));
// save/check property file for this array
File propfile = propertyFile(path, filenameStub, loadfactor,
rowdef.objectsize);
@@ -255,7 +262,9 @@
return theindex;
} else {
// open a ecotable
- return new kelondroEcoTable(f, indexRow(keylength, indexOrder),
true, EcoFSBufferSize);
+ long records = f.length() / indexRowdef.objectsize;
+ long necessaryRAM4fullTable = minimumRAM4Eco +
(indexRowdef.objectsize + 4) * records * 3 / 2;
+ return new kelondroEcoTable(f, indexRowdef,
serverMemory.request(necessaryRAM4fullTable, false), EcoFSBufferSize);
}
}
Modified: trunk/source/de/anomic/kelondro/kelondroEcoTable.java
===================================================================
--- trunk/source/de/anomic/kelondro/kelondroEcoTable.java 2008-01-19
01:07:52 UTC (rev 4344)
+++ trunk/source/de/anomic/kelondro/kelondroEcoTable.java 2008-01-19
01:50:24 UTC (rev 4345)
@@ -90,7 +90,7 @@
// initialize index and copy table
int records = file.size();
- long neededRAM4table = records * taildef.objectsize * 3 / 2;
+ long neededRAM4table = 10 * 1024 * 1024 + records *
(rowdef.objectsize + 4) * 3 / 2;
table = ((useTailCache) && (serverMemory.request(neededRAM4table,
true))) ? new kelondroRowSet(taildef, records + 1) : null;
index = new kelondroBytesIntMap(rowdef.primaryKeyLength,
rowdef.objectOrder, records + 1);
Modified: trunk/source/de/anomic/plasma/plasmaCrawlBalancer.java
===================================================================
--- trunk/source/de/anomic/plasma/plasmaCrawlBalancer.java 2008-01-19
01:07:52 UTC (rev 4344)
+++ trunk/source/de/anomic/plasma/plasmaCrawlBalancer.java 2008-01-19
01:50:24 UTC (rev 4345)
@@ -80,6 +80,7 @@
private File cacheStacksPath;
private String stackname;
private boolean top; // to
alternate between top and bottom of the file stack
+ private boolean fullram;
public static class domaccess {
long time;
@@ -100,7 +101,7 @@
}
}
- public plasmaCrawlBalancer(File cachePath, String stackname) {
+ public plasmaCrawlBalancer(File cachePath, String stackname, boolean
fullram) {
this.cacheStacksPath = cachePath;
this.stackname = stackname;
File stackFile = new File(cachePath, stackname + stackSuffix);
@@ -108,6 +109,7 @@
this.domainStacks = new HashMap<String, LinkedList<String>>();
this.urlRAMStack = new ArrayList<String>();
this.top = true;
+ this.fullram = fullram;
// create a stack for newly entered entries
if (!(cachePath.exists())) cachePath.mkdir(); // make the path
@@ -140,7 +142,7 @@
private void openFileIndex() {
cacheStacksPath.mkdirs();
- urlFileIndex = new kelondroEcoTable(new File(cacheStacksPath,
stackname + indexSuffix), plasmaCrawlEntry.rowdef, true, EcoFSBufferSize);
+ urlFileIndex = new kelondroEcoTable(new File(cacheStacksPath,
stackname + indexSuffix), plasmaCrawlEntry.rowdef, fullram, EcoFSBufferSize);
}
private void resetFileIndex() {
Modified: trunk/source/de/anomic/plasma/plasmaCrawlNURL.java
===================================================================
--- trunk/source/de/anomic/plasma/plasmaCrawlNURL.java 2008-01-19 01:07:52 UTC
(rev 4344)
+++ trunk/source/de/anomic/plasma/plasmaCrawlNURL.java 2008-01-19 01:50:24 UTC
(rev 4345)
@@ -74,10 +74,10 @@
public plasmaCrawlNURL(File cachePath) {
super();
- coreStack = new plasmaCrawlBalancer(cachePath, "urlNoticeCoreStack");
- limitStack = new plasmaCrawlBalancer(cachePath, "urlNoticeLimitStack");
+ coreStack = new plasmaCrawlBalancer(cachePath, "urlNoticeCoreStack",
true);
+ limitStack = new plasmaCrawlBalancer(cachePath, "urlNoticeLimitStack",
false);
//overhangStack = new plasmaCrawlBalancer(overhangStackFile);
- remoteStack = new plasmaCrawlBalancer(cachePath,
"urlNoticeRemoteStack");
+ remoteStack = new plasmaCrawlBalancer(cachePath,
"urlNoticeRemoteStack", true);
}
public void close() {
@@ -213,7 +213,7 @@
private plasmaCrawlEntry[] top(plasmaCrawlBalancer balancer, int count) {
// this is a filo - top
if (count > balancer.size()) count = balancer.size();
- ArrayList list = new ArrayList(count);
+ ArrayList<plasmaCrawlEntry> list = new
ArrayList<plasmaCrawlEntry>(count);
for (int i = 0; i < count; i++) {
try {
plasmaCrawlEntry entry = balancer.top(i);
@@ -226,15 +226,15 @@
return (plasmaCrawlEntry[]) list.toArray(new
plasmaCrawlEntry[list.size()]);
}
- public Iterator iterator(int stackType) {
+ public Iterator<plasmaCrawlEntry> iterator(int stackType) {
// returns an iterator of plasmaCrawlBalancerEntry Objects
try {switch (stackType) {
- case STACK_TYPE_CORE: return coreStack.iterator();
- case STACK_TYPE_LIMIT: return limitStack.iterator();
- case STACK_TYPE_REMOTE: return remoteStack.iterator();
- default: return null;
+ case STACK_TYPE_CORE: return coreStack.iterator();
+ case STACK_TYPE_LIMIT: return limitStack.iterator();
+ case STACK_TYPE_REMOTE: return remoteStack.iterator();
+ default: return null;
}} catch (IOException e) {
- return new HashSet().iterator();
+ return new HashSet<plasmaCrawlEntry>().iterator();
}
}
_______________________________________________
YaCy-svn mailing list
[email protected]
https://lists.berlios.de/mailman/listinfo/yacy-svn
