Author: grothoff
Date: 2008-02-13 22:55:07 -0700 (Wed, 13 Feb 2008)
New Revision: 6293
Modified:
GNUnet/src/applications/sqstore_mysql/mysql.c
GNUnet/src/applications/sqstore_sqlite/sqlite.c
GNUnet/todo
Log:
datastore iterator improvements
Modified: GNUnet/src/applications/sqstore_mysql/mysql.c
===================================================================
--- GNUnet/src/applications/sqstore_mysql/mysql.c 2008-02-14 05:19:17 UTC
(rev 6292)
+++ GNUnet/src/applications/sqstore_mysql/mysql.c 2008-02-14 05:55:07 UTC
(rev 6293)
@@ -1292,12 +1292,21 @@
qbind[1].buffer_type = MYSQL_TYPE_LONGLONG;
qbind[1].is_unsigned = GNUNET_YES;
qbind[1].buffer = &last_vkey;
- qbind[2].buffer_type = MYSQL_TYPE_LONG;
- qbind[2].is_unsigned = GNUNET_YES;
- qbind[2].buffer = &type;
- qbind[3].buffer_type = MYSQL_TYPE_LONG;
- qbind[3].is_unsigned = GNUNET_YES;
- qbind[3].buffer = &limit_off;
+ if (type != 0)
+ {
+ qbind[2].buffer_type = MYSQL_TYPE_LONG;
+ qbind[2].is_unsigned = GNUNET_YES;
+ qbind[2].buffer = &type;
+ qbind[3].buffer_type = MYSQL_TYPE_LONG;
+ qbind[3].is_unsigned = GNUNET_YES;
+ qbind[3].buffer = &limit_off;
+ }
+ else
+ {
+ qbind[2].buffer_type = MYSQL_TYPE_LONG;
+ qbind[2].is_unsigned = GNUNET_YES;
+ qbind[2].buffer = &limit_off;
+ }
memset (rbind, 0, sizeof (rbind));
rbind[0].buffer_type = MYSQL_TYPE_LONG;
rbind[0].buffer = &size;
@@ -1404,10 +1413,9 @@
GNUNET_mutex_unlock (lock);
}
GNUNET_free (datum);
- off++;
if (count + off == total)
last_vkey = 0; /* back to start */
- if (off == total)
+ if (count == total)
break;
}
mysql_thread_end ();
Modified: GNUnet/src/applications/sqstore_sqlite/sqlite.c
===================================================================
--- GNUnet/src/applications/sqstore_sqlite/sqlite.c 2008-02-14 05:19:17 UTC
(rev 6292)
+++ GNUnet/src/applications/sqstore_sqlite/sqlite.c 2008-02-14 05:55:07 UTC
(rev 6293)
@@ -1114,7 +1114,10 @@
unsigned int type, GNUNET_DatastoreValueIterator iter, void *closure)
{
int ret;
- int count = 0;
+ int count;
+ int total;
+ int off;
+ int limit_off;
sqlite3_stmt *stmt;
char scratch[256];
GNUNET_DatastoreValue *datum;
@@ -1129,16 +1132,67 @@
GNUNET_mutex_lock (lock);
handle = getDBHandle ();
dbh = handle->dbh;
- strcpy (scratch, "SELECT ");
- if (iter == NULL)
- strcat (scratch, "count(*)");
- else
- strcat (scratch,
- "size, type, prio, anonLevel, expire, hash, value, _ROWID_");
- strcat (scratch, " FROM gn070 WHERE hash = :1 AND _ROWID_ > :2");
+
+ strcpy (scratch, "SELECT count(*) FROM gn070 WHERE hash = :1");
if (type)
+ strcat (scratch, " AND type = :2");
+ if (sq_prepare (dbh, scratch, &stmt) != SQLITE_OK)
+ {
+ LOG_SQLITE (handle,
+ GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_USER |
+ GNUNET_GE_BULK, "sqlite_prepare");
+ GNUNET_mutex_unlock (lock);
+ return GNUNET_SYSERR;
+ }
+ ret = sqlite3_bind_blob (stmt,
+ 1,
+ key, sizeof (GNUNET_HashCode),
+ SQLITE_TRANSIENT);
+ if (type && (ret == SQLITE_OK))
+ ret = sqlite3_bind_int (stmt, 2, type);
+ if (ret != SQLITE_OK)
+ {
+ LOG_SQLITE (handle,
+ GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_USER |
+ GNUNET_GE_BULK, "sqlite_bind");
+ sqlite3_reset (stmt);
+ sqlite3_finalize (stmt);
+ GNUNET_mutex_unlock (lock);
+ return GNUNET_SYSERR;
+
+ }
+ ret = sqlite3_step (stmt);
+ if (ret != SQLITE_ROW)
+ {
+ LOG_SQLITE (handle,
+ GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_USER |
+ GNUNET_GE_BULK, "sqlite_step");
+ sqlite3_reset (stmt);
+ sqlite3_finalize (stmt);
+ GNUNET_mutex_unlock (lock);
+ return GNUNET_SYSERR;
+
+ }
+ total = sqlite3_column_int (stmt, 0);
+ sqlite3_reset (stmt);
+ sqlite3_finalize (stmt);
+ if ( (iter == NULL) ||
+ (total == 0) )
+ {
+ GNUNET_mutex_unlock (lock);
+ return total;
+ }
+
+ strcpy (scratch,
+ "SELECT size, type, prio, anonLevel, expire, hash, value, _ROWID_ "
+ "FROM gn070 WHERE hash = :1 AND _ROWID_ >= :2");
+ if (type)
strcat (scratch, " AND type = :3");
strcat (scratch, " ORDER BY _ROWID_ ASC LIMIT 1");
+ if (type)
+ strcat(scratch, " OFFSET :4");
+ else
+ strcat(scratch, " OFFSET :3");
if (sq_prepare (dbh, scratch, &stmt) != SQLITE_OK)
{
LOG_SQLITE (handle,
@@ -1149,9 +1203,14 @@
}
count = 0;
last_rowid = 0;
+ off = GNUNET_random_u32(GNUNET_RANDOM_QUALITY_WEAK, total);
while (1)
{
- ret = sqlite3_bind_blob (stmt,
+ if (count == 0)
+ limit_off = off;
+ else
+ limit_off = 0;
+ ret = sqlite3_bind_blob (stmt,
1,
key, sizeof (GNUNET_HashCode),
SQLITE_TRANSIENT);
@@ -1160,20 +1219,14 @@
if (type && (ret == SQLITE_OK))
ret = sqlite3_bind_int (stmt, 3, type);
if (ret == SQLITE_OK)
+ ret = sqlite3_bind_int (stmt, (type == 0) ? 3 : 4, limit_off);
+ if (ret == SQLITE_OK)
{
ret = sqlite3_step (stmt);
if (ret != SQLITE_ROW)
break;
- if (iter == NULL)
- {
- count = sqlite3_column_int (stmt, 0);
- sqlite3_reset (stmt);
- sqlite3_finalize (stmt);
- GNUNET_mutex_unlock (lock);
- return count;
- }
datum = assembleDatum (handle, stmt, &rkey, &rowid);
- last_rowid = rowid;
+ last_rowid = rowid + 1;
sqlite3_reset (stmt);
if (datum == NULL)
continue;
@@ -1201,6 +1254,11 @@
}
GNUNET_free (datum);
}
+ off++;
+ if (count + off == total)
+ last_rowid = 0; /* back to start */
+ if (off == total)
+ break;
}
sqlite3_reset (stmt);
sqlite3_finalize (stmt);
Modified: GNUnet/todo
===================================================================
--- GNUnet/todo 2008-02-14 05:19:17 UTC (rev 6292)
+++ GNUnet/todo 2008-02-14 05:55:07 UTC (rev 6293)
@@ -4,21 +4,11 @@
RC == Release Critical
0.7.4 [4'08] (aka "fix search"):
-- modify datastore to return diverse subsets of large response sets,
- except when processing for loopback [RC]
- IDEA: do pre-pass over result set, count # number; then LIMIT with OFFSET
- by RAND(total) for the first result and then wrap around
- to the beginning if the iterator reaches total.
- + reasonably fast for arbitrary large result sets
- - except measureable slowdown (?) for result sets of size 1?)
- + equal chance to get any result first
- + no need for threads per multi-result query
- - non-trivial code required in every sqstore implementation
- o testable, but not trivial to test fully
+- test that modified datastores return diverse subsets of large response sets
- modify gap.c to NOT do full iteration over result set (current code
might take forever; add bound in # replies (prio+1?) and
# iterations (10*(prio+1)?)done!)
-- fix pid-table assertion crash (rare, produced with linear-gap test) [RC]
+- fix pid-table assertion crash (rare, produced once (!) with linear-gap test)
[RC]
- tune GAP query planning code [RC]
- reenable and test GAP migration code [RC]
- complete IPv4/IPv6 integration of transports (http is missing!) [RC]
_______________________________________________
GNUnet-SVN mailing list
GNUnet-SVN@gnu.org
http://lists.gnu.org/mailman/listinfo/gnunet-svn
