Author: orbiter
Date: 2008-01-19 13:23:56 +0100 (Sat, 19 Jan 2008)
New Revision: 4346
Modified:
trunk/source/dbtest.java
trunk/source/de/anomic/kelondro/kelondroBytesIntMap.java
trunk/source/de/anomic/kelondro/kelondroCollectionIndex.java
trunk/source/de/anomic/kelondro/kelondroDyn.java
trunk/source/de/anomic/kelondro/kelondroEcoFS.java
trunk/source/de/anomic/kelondro/kelondroEcoTable.java
trunk/source/de/anomic/kelondro/kelondroRAMIndex.java
trunk/source/de/anomic/kelondro/kelondroSplitTable.java
trunk/source/de/anomic/plasma/plasmaCrawlBalancer.java
trunk/source/de/anomic/plasma/plasmaCrawlStacker.java
trunk/source/de/anomic/plasma/plasmaCrawlZURL.java
Log:
fixed some problems with eco tables
Modified: trunk/source/dbtest.java
===================================================================
--- trunk/source/dbtest.java 2008-01-19 01:50:24 UTC (rev 4345)
+++ trunk/source/dbtest.java 2008-01-19 12:23:56 UTC (rev 4346)
@@ -215,7 +215,7 @@
table = new kelondroSplitTable(tablepath, new
File(tablename).getName(), preload, testRow, true);
}
if (dbe.equals("kelondroEcoTable")) {
- table = new kelondroEcoTable(new File(tablename), testRow,
true, 100);
+ table = new kelondroEcoTable(new File(tablename), testRow,
kelondroEcoTable.tailCacheDenyUsage /*kelondroEcoTable.tailCacheForceUsage*/,
100);
}
if (dbe.equals("mysql")) {
table = new kelondroSQLTable("mysql", testRow);
@@ -377,6 +377,16 @@
//
// args: <number-of-writes> <number-of-reads-per-write>
<random-startpoint>
// example: kelondroFlexTable stressThreaded
/Users/admin/dbtest 500 50 0
+ /* result with svn 4346
+ kelondroFlex:
+ removed: 70, size of jcontrol set: 354, size of kcontrol
set: 354
+ Database size = 354 unique entries.
+ Execution time: open=1329, command=36234, close=17,
total=37580
+ kelondroEco:
+ removed: 70, size of jcontrol set: 354, size of kcontrol
set: 354
+ Database size = 354 unique entries.
+ Execution time: open=1324, command=34032, close=1,
total=35357
+ */
long writeCount = Long.parseLong(args[3]);
long readCount = Long.parseLong(args[4]);
long randomstart = Long.parseLong(args[5]);
Modified: trunk/source/de/anomic/kelondro/kelondroBytesIntMap.java
===================================================================
--- trunk/source/de/anomic/kelondro/kelondroBytesIntMap.java 2008-01-19
01:50:24 UTC (rev 4345)
+++ trunk/source/de/anomic/kelondro/kelondroBytesIntMap.java 2008-01-19
12:23:56 UTC (rev 4346)
@@ -76,7 +76,7 @@
public synchronized int removei(byte[] key) throws IOException {
assert (key != null);
- kelondroRow.Entry indexentry = index.remove(key, false);
+ kelondroRow.Entry indexentry = index.remove(key, true); // keeping the
order will prevent multiple re-sorts
if (indexentry == null) return -1;
return (int) indexentry.getColLong(1);
}
Modified: trunk/source/de/anomic/kelondro/kelondroCollectionIndex.java
===================================================================
--- trunk/source/de/anomic/kelondro/kelondroCollectionIndex.java
2008-01-19 01:50:24 UTC (rev 4345)
+++ trunk/source/de/anomic/kelondro/kelondroCollectionIndex.java
2008-01-19 12:23:56 UTC (rev 4346)
@@ -160,9 +160,9 @@
// initialize (new generation) index table from file
if (serverMemory.request(necessaryRAM4fullTable, false)) {
- index = new kelondroEcoTable(f, indexRowdef, true,
EcoFSBufferSize);
+ index = new kelondroEcoTable(f, indexRowdef,
kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize);
} else if (serverMemory.request(necessaryRAM4fullIndex, false)) {
- index = new kelondroEcoTable(f, indexRowdef, false,
EcoFSBufferSize);
+ index = new kelondroEcoTable(f, indexRowdef,
kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize);
} else {
index = new kelondroFlexTable(path, filenameStub + ".index",
preloadTime, indexRowdef, initialSpace, true);
}
@@ -221,7 +221,8 @@
ientry.setCol(idx_col_indexpos, aentry.index());
ientry.setCol(idx_col_lastread, t);
ientry.setCol(idx_col_lastwrote, t);
- index.addUnique(ientry); // FIXME: this should avoid
doubles
+ //index.addUnique(ientry); // FIXME: this should avoid
doubles
+ index.put(ientry);
count++;
// write a log
@@ -264,7 +265,7 @@
// open a ecotable
long records = f.length() / indexRowdef.objectsize;
long necessaryRAM4fullTable = minimumRAM4Eco +
(indexRowdef.objectsize + 4) * records * 3 / 2;
- return new kelondroEcoTable(f, indexRowdef,
serverMemory.request(necessaryRAM4fullTable, false), EcoFSBufferSize);
+ return new kelondroEcoTable(f, indexRowdef,
(serverMemory.request(necessaryRAM4fullTable, false)) ?
kelondroEcoTable.tailCacheUsageAuto : kelondroEcoTable.tailCacheDenyUsage,
EcoFSBufferSize);
}
}
@@ -978,7 +979,7 @@
kelondroRowSet collection = new kelondroRowSet(this.payloadrow,
arrayrow, 1); // FIXME: this does not yet work with different rowdef in case of
several rowdef.objectsize()
if ((!(index.row().objectOrder.wellformed(indexkey))) ||
(index.row().objectOrder.compare(arraykey, indexkey) != 0)) {
// check if we got the right row; this row is wrong. Fix it:
- index.remove(indexkey, true); // the wrong row cannot be fixed
+ index.remove(indexkey, false); // the wrong row cannot be fixed
// store the row number in the index; this may be a double-entry,
but better than nothing
kelondroRow.Entry indexEntry = index.row().newEntry();
indexEntry.setCol(idx_col_key, arrayrow.getColBytes(0));
Modified: trunk/source/de/anomic/kelondro/kelondroDyn.java
===================================================================
--- trunk/source/de/anomic/kelondro/kelondroDyn.java 2008-01-19 01:50:24 UTC
(rev 4345)
+++ trunk/source/de/anomic/kelondro/kelondroDyn.java 2008-01-19 12:23:56 UTC
(rev 4346)
@@ -99,10 +99,10 @@
if (file.isDirectory()) {
fbi = new kelondroFlexTable(file.getParentFile(),
file.getName(), 10000, rowdef, 0, resetOnFail);
} else {
- fbi = new kelondroEcoTable(file, rowdef, false,
EcoFSBufferSize);
+ fbi = new kelondroEcoTable(file, rowdef,
kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize);
}
} else {
- fbi = new kelondroEcoTable(file, rowdef, false,
EcoFSBufferSize);
+ fbi = new kelondroEcoTable(file, rowdef,
kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize);
}
}
this.index = (useObjectCache) ? (kelondroIndex) new kelondroCache(fbi)
: fbi;
Modified: trunk/source/de/anomic/kelondro/kelondroEcoFS.java
===================================================================
--- trunk/source/de/anomic/kelondro/kelondroEcoFS.java 2008-01-19 01:50:24 UTC
(rev 4345)
+++ trunk/source/de/anomic/kelondro/kelondroEcoFS.java 2008-01-19 12:23:56 UTC
(rev 4346)
@@ -485,7 +485,14 @@
System.out.println("size = " + t.size());
t.clean(t.size() - 2);
t.cleanLast();
- System.out.println("size = " + t.size());
+ long start = System.currentTimeMillis();
+ long c = 0;
+ for (int i = 0; i < 100000; i++) {
+ c = t.size();
+ }
+ System.out.println("size() needs " + ((System.currentTimeMillis()
- start) / 100) + " nanoseconds");
+ System.out.println("size = " + c);
+
t.close();
} catch (IOException e) {
e.printStackTrace();
Modified: trunk/source/de/anomic/kelondro/kelondroEcoTable.java
===================================================================
--- trunk/source/de/anomic/kelondro/kelondroEcoTable.java 2008-01-19
01:50:24 UTC (rev 4345)
+++ trunk/source/de/anomic/kelondro/kelondroEcoTable.java 2008-01-19
12:23:56 UTC (rev 4346)
@@ -54,13 +54,17 @@
// static tracker objects
private static TreeMap<String, kelondroEcoTable> tableTracker = new
TreeMap<String, kelondroEcoTable>();
+ public static final int tailCacheDenyUsage = 0;
+ public static final int tailCacheForceUsage = 1;
+ public static final int tailCacheUsageAuto = 2;
+
private kelondroRowSet table;
private kelondroBytesIntMap index;
private kelondroBufferedEcoFS file;
private kelondroRow rowdef, taildef;
private int buffersize;
- public kelondroEcoTable(File tablefile, kelondroRow rowdef, boolean
useTailCache, int buffersize) {
+ public kelondroEcoTable(File tablefile, kelondroRow rowdef, int
useTailCache, int buffersize) {
this.rowdef = rowdef;
this.buffersize = buffersize;
assert rowdef.primaryKeyIndex == 0;
@@ -91,7 +95,9 @@
// initialize index and copy table
int records = file.size();
long neededRAM4table = 10 * 1024 * 1024 + records *
(rowdef.objectsize + 4) * 3 / 2;
- table = ((useTailCache) && (serverMemory.request(neededRAM4table,
true))) ? new kelondroRowSet(taildef, records + 1) : null;
+ table = ((useTailCache == tailCacheForceUsage) ||
+ ((useTailCache == tailCacheUsageAuto) &&
(serverMemory.request(neededRAM4table, true)))) ?
+ new kelondroRowSet(taildef, records + 1) : null;
index = new kelondroBytesIntMap(rowdef.primaryKeyLength,
rowdef.objectOrder, records + 1);
// read all elements from the file into the copy table
@@ -104,6 +110,7 @@
// write the key into the index table
System.arraycopy(record, 0, key, 0, rowdef.primaryKeyLength);
index.addi(key, i);
+ //index.puti(key, i);
// write the tail into the table
if (table != null) table.addUnique(taildef.newEntry(record,
rowdef.primaryKeyLength, true));
@@ -116,6 +123,11 @@
e.printStackTrace();
throw new kelondroException(e.getMessage());
}
+ try {
+ assert file.size() == index.size() : "file.size() = " +
file.size() + ", index.size() = " + index.size();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
// track this table
tableTracker.put(tablefile.toString(), this);
@@ -156,7 +168,7 @@
}
public synchronized void addUnique(Entry row) throws IOException {
- assert (file.size() == index.size());
+ assert file.size() == index.size() : "file.size() = " + file.size() +
", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
int i = file.size();
index.addi(row.getPrimaryKeyBytes(), i);
@@ -165,13 +177,16 @@
table.addUnique(taildef.newEntry(row.bytes(),
rowdef.primaryKeyLength, true));
}
file.put(i, row.bytes(), 0);
+ assert file.size() == index.size() : "file.size() = " + file.size() +
", index.size() = " + index.size();
}
public synchronized void addUniqueMultiple(List<Entry> rows) throws
IOException {
+ assert file.size() == index.size() : "file.size() = " + file.size() +
", index.size() = " + index.size();
Iterator<Entry> i = rows.iterator();
while (i.hasNext()) {
addUnique(i.next());
}
+ assert file.size() == index.size() : "file.size() = " + file.size() +
", index.size() = " + index.size();
}
public void close() {
@@ -188,7 +203,7 @@
}
public synchronized Entry get(byte[] key) throws IOException {
- assert (file.size() == index.size());
+ assert file.size() == index.size() : "file.size() = " + file.size() +
", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
int i = index.geti(key);
if (i == -1) return null;
@@ -204,13 +219,13 @@
System.arraycopy(key, 0, b, 0, key.length);
System.arraycopy(v.bytes(), 0, b, rowdef.primaryKeyLength,
rowdef.objectsize - rowdef.primaryKeyLength);
}
- assert (file.size() == index.size());
+ assert file.size() == index.size() : "file.size() = " + file.size() +
", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
return rowdef.newEntry(b);
}
public synchronized boolean has(byte[] key) throws IOException {
- assert (file.size() == index.size());
+ assert file.size() == index.size() : "file.size() = " + file.size() +
", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
return index.geti(key) >= 0;
}
@@ -224,7 +239,7 @@
}
public synchronized Entry put(Entry row) throws IOException {
- assert (file.size() == index.size());
+ assert file.size() == index.size() : "file.size() = " + file.size() +
", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
int i = index.geti(row.getPrimaryKeyBytes());
if (i == -1) {
@@ -247,7 +262,7 @@
table.set(i, taildef.newEntry(row.bytes(),
rowdef.primaryKeyLength, true));
file.put(i, row.bytes(), 0);
}
- assert (file.size() == index.size());
+ assert file.size() == index.size() : "file.size() = " + file.size() +
", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
// return old value
return rowdef.newEntry(b);
@@ -258,14 +273,16 @@
}
public synchronized void putMultiple(List<Entry> rows) throws IOException {
+ assert file.size() == index.size() : "file.size() = " + file.size() +
", index.size() = " + index.size();
Iterator<Entry> i = rows.iterator();
while (i.hasNext()) {
put(i.next());
}
+ assert file.size() == index.size() : "file.size() = " + file.size() +
", index.size() = " + index.size();
}
public synchronized Entry remove(byte[] key, boolean keepOrder) throws
IOException {
- assert (file.size() == index.size());
+ assert file.size() == index.size() : "file.size() = " + file.size() +
", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
assert keepOrder == false; // this class cannot keep the order during
a remove
int i = index.geti(key);
@@ -311,13 +328,13 @@
assert ((table == null) || (table.size() == index.size())) :
"table.size() = " + table.size() + ", index.size() = " + index.size();
}
}
- assert (file.size() == index.size());
+ assert file.size() == index.size() : "file.size() = " + file.size() +
", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
return rowdef.newEntry(b);
}
public synchronized Entry removeOne() throws IOException {
- assert (file.size() == index.size());
+ assert file.size() == index.size() : "file.size() = " + file.size() +
", index.size() = " + index.size();
assert ((table == null) || (table.size() == index.size()));
byte[] le = new byte[rowdef.objectsize];
file.cleanLast(le, 0);
@@ -325,6 +342,7 @@
int i = index.removei(lr.getPrimaryKeyBytes());
assert i >= 0;
table.removeRow(i, false);
+ assert file.size() == index.size() : "file.size() = " + file.size() +
", index.size() = " + index.size();
return lr;
}
@@ -428,10 +446,10 @@
}
- public static kelondroIndex testTable(File f, String testentities) throws
IOException {
+ public static kelondroIndex testTable(File f, String testentities, int
testcase) throws IOException {
if (f.exists()) f.delete();
kelondroRow rowdef = new kelondroRow("byte[] a-4, byte[] b-4",
kelondroNaturalOrder.naturalOrder, 0);
- kelondroIndex tt = new kelondroEcoTable(f, rowdef, true, 100);
+ kelondroIndex tt = new kelondroEcoTable(f, rowdef, testcase, 100);
byte[] b;
kelondroRow.Entry row = rowdef.newEntry();
for (int i = 0; i < testentities.length(); i++) {
@@ -443,7 +461,7 @@
return tt;
}
- public static void bigtest(int elements, File testFile) {
+ public static void bigtest(int elements, File testFile, int testcase) {
System.out.println("starting big test with " + elements + "
elements:");
long start = System.currentTimeMillis();
String[] s = kelondroTree.permutations(elements);
@@ -452,13 +470,13 @@
for (int i = 0; i < s.length; i++) {
System.out.println("*** probing tree " + i + " for permutation
" + s[i]);
// generate tree and delete elements
- tt = testTable(testFile, s[i]);
+ tt = testTable(testFile, s[i], testcase);
if (kelondroTree.countElements(tt) != tt.size()) {
System.out.println("wrong size for " + s[i]);
}
tt.close();
for (int j = 0; j < s.length; j++) {
- tt = testTable(testFile, s[i]);
+ tt = testTable(testFile, s[i], testcase);
// delete by permutation j
for (int elt = 0; elt < s[j].length(); elt++) {
tt.remove(kelondroTree.testWord(s[j].charAt(elt)),
false);
@@ -479,7 +497,10 @@
public static void main(String[] args) {
// open a file, add one entry and exit
File f = new File(args[0]);
- bigtest(5, f);
+ System.out.println("========= Testcase: no tail cache:");
+ bigtest(5, f, tailCacheDenyUsage);
+ System.out.println("========= Testcase: with tail cache:");
+ bigtest(5, f, tailCacheForceUsage);
/*
kelondroRow row = new kelondroRow("byte[] key-4, byte[] x-5",
kelondroNaturalOrder.naturalOrder, 0);
try {
Modified: trunk/source/de/anomic/kelondro/kelondroRAMIndex.java
===================================================================
--- trunk/source/de/anomic/kelondro/kelondroRAMIndex.java 2008-01-19
01:50:24 UTC (rev 4345)
+++ trunk/source/de/anomic/kelondro/kelondroRAMIndex.java 2008-01-19
12:23:56 UTC (rev 4346)
@@ -123,6 +123,7 @@
}
public synchronized kelondroRow.Entry remove(byte[] key, boolean
keepOrder) throws IOException {
+ assert keepOrder == true; // if this is false, the index must be
re-ordered so many times which will cause a major CPU usage
finishInitialization();
// if the new entry is within the initialization part, just delete it
kelondroRow.Entry indexentry = index0.remove(key, keepOrder);
Modified: trunk/source/de/anomic/kelondro/kelondroSplitTable.java
===================================================================
--- trunk/source/de/anomic/kelondro/kelondroSplitTable.java 2008-01-19
01:50:24 UTC (rev 4345)
+++ trunk/source/de/anomic/kelondro/kelondroSplitTable.java 2008-01-19
12:23:56 UTC (rev 4346)
@@ -118,7 +118,7 @@
// this is a kelonodroFlex table
table = new kelondroCache(new kelondroFlexTable(path, maxf,
preloadTime, rowdef, 0, resetOnFail));
} else {
- table = new kelondroEcoTable(f, rowdef, false,
EcoFSBufferSize);
+ table = new kelondroEcoTable(f, rowdef,
kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize);
}
tables.put(date, table);
}
@@ -228,13 +228,13 @@
table = new kelondroFlexTable(path, tablename + "." +
suffix, -1, rowdef, 0, true);
} else {
// open a eco table
- table = new kelondroEcoTable(f, rowdef, false,
EcoFSBufferSize);
+ table = new kelondroEcoTable(f, rowdef,
kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize);
}
} else {
// make new table
if (serverMemory.request(minimumRAM4Eco, true)) {
// enough memory for a ecoTable
- table = new kelondroEcoTable(f, rowdef, false,
EcoFSBufferSize);
+ table = new kelondroEcoTable(f, rowdef,
kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize);
} else {
// use the flex table
table = new kelondroFlexTable(path, tablename + "." +
suffix, -1, rowdef, 0, true);
@@ -272,7 +272,7 @@
// make new table
if (serverMemory.request(minimumRAM4Eco, true)) {
// enough memory for a ecoTable
- table = new kelondroEcoTable(new File(path, tablename + "." +
suffix), rowdef, false, EcoFSBufferSize);
+ table = new kelondroEcoTable(new File(path, tablename + "." +
suffix), rowdef, kelondroEcoTable.tailCacheDenyUsage, EcoFSBufferSize);
} else {
// use the flex table
table = new kelondroFlexTable(path, tablename + "." + suffix,
-1, rowdef, 0, true);
Modified: trunk/source/de/anomic/plasma/plasmaCrawlBalancer.java
===================================================================
--- trunk/source/de/anomic/plasma/plasmaCrawlBalancer.java 2008-01-19
01:50:24 UTC (rev 4345)
+++ trunk/source/de/anomic/plasma/plasmaCrawlBalancer.java 2008-01-19
12:23:56 UTC (rev 4346)
@@ -142,7 +142,7 @@
private void openFileIndex() {
cacheStacksPath.mkdirs();
- urlFileIndex = new kelondroEcoTable(new File(cacheStacksPath,
stackname + indexSuffix), plasmaCrawlEntry.rowdef, fullram, EcoFSBufferSize);
+ urlFileIndex = new kelondroEcoTable(new File(cacheStacksPath,
stackname + indexSuffix), plasmaCrawlEntry.rowdef, (fullram) ?
kelondroEcoTable.tailCacheUsageAuto : kelondroEcoTable.tailCacheDenyUsage,
EcoFSBufferSize);
}
private void resetFileIndex() {
Modified: trunk/source/de/anomic/plasma/plasmaCrawlStacker.java
===================================================================
--- trunk/source/de/anomic/plasma/plasmaCrawlStacker.java 2008-01-19
01:50:24 UTC (rev 4345)
+++ trunk/source/de/anomic/plasma/plasmaCrawlStacker.java 2008-01-19
12:23:56 UTC (rev 4346)
@@ -302,7 +302,7 @@
cacheStacksPath.mkdirs();
File f = new File(cacheStacksPath, stackfile);
try {
- this.urlEntryCache = new kelondroEcoTable(f,
plasmaCrawlEntry.rowdef, true, EcoFSBufferSize);
+ this.urlEntryCache = new kelondroEcoTable(f,
plasmaCrawlEntry.rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize);
//this.urlEntryCache = new kelondroCache(new
kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime,
plasmaCrawlEntry.rowdef, 0, true));
} catch (Exception e) {
e.printStackTrace();
@@ -310,7 +310,7 @@
f.delete();
//kelondroFlexTable.delete(cacheStacksPath, newCacheName);
try {
- this.urlEntryCache = new kelondroEcoTable(f,
plasmaCrawlEntry.rowdef, true, EcoFSBufferSize);
+ this.urlEntryCache = new kelondroEcoTable(f,
plasmaCrawlEntry.rowdef, kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize);
//this.urlEntryCache = new kelondroCache(new
kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime,
plasmaCrawlEntry.rowdef, 0, true));
} catch (Exception ee) {
ee.printStackTrace();
Modified: trunk/source/de/anomic/plasma/plasmaCrawlZURL.java
===================================================================
--- trunk/source/de/anomic/plasma/plasmaCrawlZURL.java 2008-01-19 01:50:24 UTC
(rev 4345)
+++ trunk/source/de/anomic/plasma/plasmaCrawlZURL.java 2008-01-19 12:23:56 UTC
(rev 4346)
@@ -69,7 +69,7 @@
if (f.isDirectory()) kelondroFlexTable.delete(cachePath,
tablename); else f.delete();
}
}
- urlIndex = new kelondroEcoTable(f, rowdef, true, EcoFSBufferSize);
+ urlIndex = new kelondroEcoTable(f, rowdef,
kelondroEcoTable.tailCacheUsageAuto, EcoFSBufferSize);
//urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef,
0, true);
}
_______________________________________________
YaCy-svn mailing list
[email protected]
https://lists.berlios.de/mailman/listinfo/yacy-svn
