Update of /cvsroot/monetdb/pathfinder/modules/pftijah
In directory sc8-pr-cvs7.sourceforge.net:/tmp/cvs-serv9174
Modified Files:
Tag: XQuery_0-16
pftijah.mx
Log Message:
- Implement the indexing half of locking. The locking from the milprint_summer
generated script still has to be implemented.
- print 'deprecated' messages for a couple of deprecated functions.
Index: pftijah.mx
===================================================================
RCS file: /cvsroot/monetdb/pathfinder/modules/pftijah/pftijah.mx,v
retrieving revision 1.92.2.7
retrieving revision 1.92.2.8
diff -u -d -r1.92.2.7 -r1.92.2.8
--- pftijah.mx 6 Feb 2007 14:38:39 -0000 1.92.2.7
+++ pftijah.mx 14 Feb 2007 16:14:56 -0000 1.92.2.8
@@ -152,6 +152,10 @@
const SIZE_ENTITY := 1;
const SIZE_TERM := 2;
+# locks
+var tj_adm_lock := lock_create(); # tijah top administration lock
+var tj_coll_lockbat := new(str,lock); # locks for all active collections
+
# Comparisons
const GREATER := 10;
const LESS := 11;
@@ -215,7 +219,10 @@
"pftijah");
PROC tj_ping() : void :=
{
- printf("# *** ping *****\n");
+ lock_set(tj_adm_lock);
+ var err := CATCH({printf("# *** ping *****\n");} );
+ lock_unset(tj_adm_lock);
+ if (not(isnil(err))) ERROR(err);
}
ADDHELP("tj_init_global", "flokstra & rode", "Jan 2007",
@@ -227,8 +234,10 @@
"pftijah");
PROC tj_init_global(BAT[str,str] param) : void :=
{
- if ( verbose ) printf("#TJ: tj_init_global() called.\n");
- [EMAIL PROTECTED]() {
+ lock_set(tj_adm_lock);
+ var err := CATCH({
+ if ( verbose ) printf("#TJ: tj_init_global() called.\n");
+ [EMAIL PROTECTED]() {
if ( verbose ) printf("#TJ: tj_init_global():param[%s]=\"%s\"\n",$h,$t);
if ( $h = "term_frag" ) {
# incomplete, handle term frag
@@ -237,10 +246,13 @@
} else {
ERROR("# tj_init_global() unkonwn parameter [%s].\n",$h);
}
- }
- new(oid,str).persists(true).bbpname("tj_globalTerms");
- new(oid,str).persists(true).bbpname("tj_globalTags");
- new(oid,str).persists(true).bbpname("tj_collName");
+ }
+ new(oid,str).persists(true).bbpname("tj_globalTerms");
+ new(oid,str).persists(true).bbpname("tj_globalTags");
+ new(oid,str).persists(true).bbpname("tj_collName");
+ });
+ lock_unset(tj_adm_lock);
+ if (not(isnil(err))) ERROR(err);
}
PROC tj_init_global() : void :=
@@ -255,15 +267,41 @@
"pftijah");
PROC tj_delete_global() : void :=
{
- if ( verbose ) printf("#TJ: tj_delete_global() called.\n");
- #
- bat("tj_collName")@batloop(){
- tj_delete_collection($t);
- }
- bat("tj_globalTerms").persists(false);
- bat("tj_globalTags").persists(false);
- bat("tj_collName").persists(false);
- commit();
+ lock_set(tj_adm_lock);
+ var err := CATCH({
+ if ( verbose ) printf("#TJ: tj_delete_global() called.\n");
+ #
+ bat("tj_collName")@batloop(){
+ tj_delete_collection($t,false);
+ }
+ bat("tj_globalTerms").persists(false);
+ bat("tj_globalTags").persists(false);
+ bat("tj_collName").persists(false);
+ });
+ lock_unset(tj_adm_lock);
+ if (not(isnil(err))) ERROR(err);
+}
+
+# protext the lookup/insert of the per-collection-locks
+
+var tj_coll_mutex := lock_create();
+
+PROC tj_get_collection_lock(str colname) : lock
+{
+ var collection_lock;
+
+ lock_set(tj_coll_mutex);
+ var err := CATCH({
+ if (tj_coll_lockbat.exist(colname))
+ collection_lock := tj_coll_lockbat.find(colname);
+ else {
+ collection_lock := lock_create();
+ tj_coll_lockbat.insert(colname,collection_lock);
+ }
+ });
+ lock_unset(tj_coll_mutex);
+ if (not(isnil(err))) ERROR(err);
+ return collection_lock;
}
ADDHELP("tj_init_collection", "flokstra & rode", "Jan 2007",
@@ -280,62 +318,85 @@
"pftijah");
PROC tj_init_collection(str collName, BAT[str,str] param) : void :=
{
- if ( verbose ) printf("#TJ: tj_init_collection(\"%s\")
called.\n",collName);
+ var coll_lock;
- if (not(view_bbp_name().reverse().exist("tj_collName"))) {
- tj_init_global(); # just in case
- }
- if (bat("tj_collName").reverse().exist(collName)) {
- ERROR("tj_init_collection, pftijah collection already exists:
%s\n",collName);
- }
- new(void,str).seqbase([EMAIL PROTECTED]).persists(true).bbpname("tj_" +
collName + "_doc_name");
- new(void,int).seqbase([EMAIL PROTECTED]).persists(true).bbpname("tj_" +
collName + "_doc_firstpre");
- new(str,str).persists(true).bbpname("tj_" + collName + "_param");
-
- new(void,oid).seqbase([EMAIL PROTECTED]).persists(true).bbpname("tj_" +
collName + "_tid1");
- new(void,int).seqbase([EMAIL PROTECTED]).persists(true).bbpname("tj_" +
collName + "_size1");
- # bat contains the start oid of every tid/size frag. Head is postfix
- # string to _tid/_size. Normally "", "2", "3"
- new(void,oid).seqbase([EMAIL PROTECTED]).persists(true).bbpname("tj_" +
collName + "_fragments");
- bat("tj_" + collName + "_fragments").append([EMAIL PROTECTED]);
- new(oid,oid).persists(true).bbpname("tj_" + collName + "_pfpre");
-
- var coll_oid;
- if (bat("tj_collName").count() = 0) { coll_oid := [EMAIL PROTECTED]; }
- else { coll_oid := oid(int(bat("tj_collName").reverse().max())
+ 1); }
- bat("tj_collName").insert(coll_oid, collName);
#
- # now read the param file
+ # first handle the global administration stuff
#
- var stemmer := "nostemming";
- var tagfilter := "";
+ lock_set(tj_adm_lock);
+ var err := CATCH({
+ if ( verbose ) printf("#TJ: tj_init_collection(\"%s\")
called.\n",collName);
- [EMAIL PROTECTED]() {
- if ( verbose ) printf("#TJ: tj_init_global():param[%s]=\"%s\"\n",$h,$t);
- if ( $h = "stemmer" ) {
- stemmer := $t;
- } else if ( $h = "fragmentSize" ) {
- bat("tj_" + collName + "_param").insert($h,$t);
- } else if ( $h = "tagFilter" ) {
- tagfilter := $t;
- } else {
- ERROR("# tj_init_global() unkonwn parameter [%s].\n",$h);
- }
- }
- #
- # now set the parameters for this collection
+ if (not(view_bbp_name().reverse().exist("tj_collName"))) {
+ tj_init_global(); # just in case
+ }
+ if (bat("tj_collName").reverse().exist(collName)) {
+ ERROR("tj_init_collection, pftijah collection already exists:
%s\n",collName);
+ }
+ #
+ var coll_oid;
+ if (bat("tj_collName").count() = 0) {
+ coll_oid := [EMAIL PROTECTED];
+ } else {
+ coll_oid := oid(int(bat("tj_collName").reverse().max()) + 1);
+ }
+ bat("tj_collName").insert(coll_oid, collName);
+ #
+ coll_lock := tj_get_collection_lock(collName);
+ });
+ lock_unset(tj_adm_lock);
+ if (not(isnil(err))) ERROR(err);
#
- bat("tj_" + collName + "_param").insert("_version","1.01");
- bat("tj_" + collName + "_param").insert("name",collName);
- bat("tj_" + collName + "_param").insert("height","0");
- bat("tj_" + collName + "_param").insert("stemmer",stemmer);
- bat("tj_" + collName + "_param").insert("tagFilter",tagfilter);
- bat("tj_" + collName + "_param").insert("preExpansion","4");
- bat("tj_" + collName + "_param").insert("status","building");
- bat("tj_" + collName + "_param").insert("_last_tijahPre","1");
- bat("tj_" + collName + "_param").insert("_last_finalizedPre","0");
+ # now the collection stuff
#
- commit();
+ lock_set(coll_lock);
+ var err := CATCH({
+ new(void,str).seqbase([EMAIL PROTECTED]).persists(true).bbpname("tj_" +
collName + "_doc_name");
+ new(void,int).seqbase([EMAIL PROTECTED]).persists(true).bbpname("tj_" +
collName + "_doc_firstpre");
+ new(str,str).persists(true).bbpname("tj_" + collName + "_param");
+
+ new(void,oid).seqbase([EMAIL PROTECTED]).persists(true).bbpname("tj_" +
collName + "_tid1");
+ new(void,int).seqbase([EMAIL PROTECTED]).persists(true).bbpname("tj_" +
collName + "_size1");
+ # bat contains the start oid of every tid/size frag. Head is postfix
+ # string to _tid/_size. Normally "", "2", "3"
+ new(void,oid).seqbase([EMAIL PROTECTED]).persists(true).bbpname("tj_" +
collName + "_fragments");
+ bat("tj_" + collName + "_fragments").append([EMAIL PROTECTED]);
+ new(oid,oid).persists(true).bbpname("tj_" + collName + "_pfpre");
+
+ #
+ # now read the param file
+ #
+ var stemmer := "nostemming";
+ var tagfilter := "";
+
+ [EMAIL PROTECTED]() {
+ if ( verbose ) printf("#TJ:
tj_init_global():param[%s]=\"%s\"\n",$h,$t);
+ if ( $h = "stemmer" ) {
+ stemmer := $t;
+ } else if ( $h = "fragmentSize" ) {
+ bat("tj_" + collName + "_param").insert($h,$t);
+ } else if ( $h = "tagFilter" ) {
+ tagfilter := $t;
+ } else {
+ ERROR("# tj_init_global() unkonwn parameter [%s].\n",$h);
+ }
+ }
+ #
+ # now set the parameters for this collection
+ #
+ bat("tj_" + collName + "_param").insert("_version","1.01");
+ bat("tj_" + collName + "_param").insert("name",collName);
+ bat("tj_" + collName + "_param").insert("height","0");
+ bat("tj_" + collName + "_param").insert("stemmer",stemmer);
+ bat("tj_" + collName + "_param").insert("tagFilter",tagfilter);
+ bat("tj_" + collName + "_param").insert("preExpansion","4");
+ bat("tj_" + collName + "_param").insert("status","building");
+ bat("tj_" + collName + "_param").insert("_last_tijahPre","1");
+ bat("tj_" + collName + "_param").insert("_last_finalizedPre","0");
+ #
+ });
+ lock_unset(coll_lock);
+ if (not(isnil(err))) ERROR(err);
}
ADDHELP("tj_delete_collection", "flokstra & rode", "Jan 2007",
@@ -346,6 +407,39 @@
"pftijah");
PROC tj_delete_collection(str collName) : void
{
+ tj_delete_collection(collName,true);
+}
+
+PROC tj_delete_collection(str collName, bit doAdmLock) : void
+{
+ var coll_lock;
+
+ #
+ # first handle the global administration stuff
+ #
+ if ( doAdmLock ) lock_set(tj_adm_lock);
+ var err := CATCH({
+ if ( verbose ) printf("#TJ: tj_delete_collection(\"%s\")
called.\n",collName);
+
+ if (not(view_bbp_name().reverse().exist("tj_collName"))) {
+ ERROR("tj_delete_collection: pftijah not initialized.\n");
+ }
+ if (not(bat("tj_collName").reverse().exist(collName))) {
+ ERROR("tj_delete_collection, pftijah collection does not exist:
%s\n",collName);
+ }
+ #
+ var coll_oid := bat("tj_collName").reverse().find(collName);
+ bat("tj_collName").delete(coll_oid);
+ #
+ coll_lock := tj_get_collection_lock(collName);
+ });
+ if ( doAdmLock ) lock_unset(tj_adm_lock);
+ if (not(isnil(err))) ERROR(err);
+ #
+ # now remove the collection stuff
+ #
+ lock_set(coll_lock);
+ var err := CATCH({
bat("tj_" + collName + "_doc_name").persists(false);
bat("tj_" + collName + "_doc_firstpre").persists(false);
bat("tj_" + collName + "_param").persists(false);
@@ -362,10 +456,9 @@
bat("tj_" + collName + "_TagIndex").persists(false);
bat("tj_" + collName + "_Tags").persists(false);
}
-
- var coll_oid := bat("tj_collName").reverse().find(collName);
- bat("tj_collName").delete(coll_oid);
- commit();
+ });
+ lock_unset(coll_lock);
+ if (not(isnil(err))) ERROR(err);
}
# internal method which return all relevant data about a collection in a
@@ -413,15 +506,21 @@
"pftijah");
PROC tj_add2collection(str collName, str uri_loc, str uri_name, bit store) :
void
{
- var t_start := usec();
- bat("tj_globalTerms").access(BAT_WRITE);
- bat("tj_globalTags").access(BAT_WRITE);
-
_tj_add2collection(collName,_tj_collection(collName),uri_loc,uri_name,store);
- _tj_finalize_collection(collName);
- if ( timing ) {
- var ms := (usec()-t_start)/1000;
- printf("#C[%s]:tj_add2collection(): + aggregate time =
%lld.%03llds.\n",collName,/(ms,1000),%(ms,1000));
- }
+ var coll_lock := tj_get_collection_lock(collName);
+ lock_set(coll_lock);
+ var err := CATCH({
+ var t_start := usec();
+ bat("tj_globalTerms").access(BAT_WRITE);
+ bat("tj_globalTags").access(BAT_WRITE);
+
_tj_add2collection(collName,_tj_collection(collName),uri_loc,uri_name,store);
+ _tj_finalize_collection(collName);
+ if ( timing ) {
+ var ms := (usec()-t_start)/1000;
+ printf("#C[%s]:tj_add2collection(): + aggregate time =
%lld.%03llds.\n",collName,/(ms,1000),%(ms,1000));
+ }
+ });
+ lock_unset(coll_lock);
+ if (not(isnil(err))) ERROR(err);
}
ADDHELP("tj_add2collection", "flokstra & rode", "Jan 2007",
@@ -438,18 +537,24 @@
"pftijah");
PROC tj_add2collection(str collName, BAT[str,str] uri, bit store) : void
{
- var t_start := usec();
- bat("tj_globalTerms").access(BAT_WRITE);
- bat("tj_globalTags").access(BAT_WRITE);
- # var cb := _tj_collection(collName);
- [EMAIL PROTECTED]() {
- _tj_add2collection(collName,_tj_collection(collName),$h,$t,store);
- }
- _tj_finalize_collection(collName);
- if ( timing ) {
- var ms := (usec()-t_start)/1000;
- printf("#C[%s]:tj_add2collection(BAT): + aggregate time =
%lld.%03llds.\n",collName,/(ms,1000),%(ms,1000));
- }
+ var coll_lock := tj_get_collection_lock(collName);
+ lock_set(coll_lock);
+ var err := CATCH({
+ var t_start := usec();
+ bat("tj_globalTerms").access(BAT_WRITE);
+ bat("tj_globalTags").access(BAT_WRITE);
+ # var cb := _tj_collection(collName);
+ [EMAIL PROTECTED]() {
+ _tj_add2collection(collName,_tj_collection(collName),$h,$t,store);
+ }
+ _tj_finalize_collection(collName);
+ if ( timing ) {
+ var ms := (usec()-t_start)/1000;
+ printf("#C[%s]:tj_add2collection(BAT): + aggregate time =
%lld.%03llds.\n",collName,/(ms,1000),%(ms,1000));
+ }
+ });
+ lock_unset(coll_lock);
+ if (not(isnil(err))) ERROR(err);
}
PROC _timestr(lng v) : str
@@ -519,7 +624,13 @@
"pftijah");
PROC tj_finalize_collection(str collName) : void
{
- _tj_finalize_collection(collName);
+ var coll_lock := tj_get_collection_lock(collName);
+ lock_set(coll_lock);
+ var err := CATCH({
+ _tj_finalize_collection(collName);
+ });
+ lock_unset(coll_lock);
+ if (not(isnil(err))) ERROR(err);
}
# internal finalize function
@@ -589,6 +700,9 @@
PROC tj_dump_collection(str collName) : void
{
+ var coll_lock := tj_get_collection_lock(collName);
+ lock_set(coll_lock);
+ var err := CATCH({
bat("tj_" + collName + "_param").print();
bat("tj_" + collName + "_doc_name").print();
bat("tj_" + collName + "_doc_firstpre").print();
@@ -600,6 +714,9 @@
print(bat("tj_" + collName + "_pfpre"));
print(bat("tj_" + collName + "_fragments"));
# print(bat("tj_" + collName + "_height"));
+ });
+ lock_unset(coll_lock);
+ if (not(isnil(err))) ERROR(err);
}
#####################################################################
@@ -612,7 +729,7 @@
"PARAMETERS:\n\
- the name of the collection to be set.\n\
DESCRIPTION:\n\
-debugging, when this help is removed Mtest/Mapi fails ",
+function should be obsolete in the future (no locking impl) ",
"pftijah");
# SUGAR: this function should not be necessary in the future but it
# first has to be removed from the generated scripts.
@@ -633,7 +750,7 @@
}
}
-# INCOMPLETE: henning, what should I do about this.
+# INCOMPLETE: henning, what should I do about this (no locking impl).
PROC tj_setBackgroundCollName(str name) : void :=
{
if (not(view_bbp_name().reverse().exist("tj_collName"))) {
@@ -2273,7 +2390,6 @@
#"Order regions in descending order based on their score values.",
#"TIJAH");
-
##########################################
# Collection Management Functions
#
@@ -2282,12 +2398,14 @@
# SUGAR, REMOVE IN FUTURE
PROC tj_global_init() : void
{
+ printf("WARNING: the use of function tj_global_init() is
deprecated.\n");
tj_init_global();
}
# SUGAR, REMOVE IN FUTURE
PROC tj_coll_init(str collName, str stemmer) : void
{
+ printf("WARNING: the use of function tj_coll_init() is deprecated.\n");
var collpar := new(str,str);
collpar.insert("stemmer",stemmer);
tj_init_collection(collName,collpar);
@@ -2296,12 +2414,14 @@
# SUGAR, REMOVE IN FUTURE
PROC tj_global_remove() : void
{
+ printf("WARNING: the use of function tj_global_remove() is
deprecated.\n");
tj_delete_global();
}
# SUGAR, REMOVE IN FUTURE
PROC tj_coll_remove(str collName) : void
{
+ printf("WARNING: the use of function tj_coll_remove() is
deprecated.\n");
tj_delete_collection(collName);
commit();
}
@@ -2309,18 +2429,21 @@
# SUGAR, REMOVE IN FUTURE
PROC tj_collection(str collName) : BAT[void,bat]
{
+ printf("WARNING: the use of function tj_collection() is
deprecated.\n");
return _tj_collection(collName);
}
# SUGAR, REMOVE IN FUTURE
PROC tj_url2Collection(str doc, str url, str selTagList) : void
{
+ printf("WARNING: the use of function tj_url2Collection() is
deprecated.\n");
tj_url2Collection(collName, doc, url, selTagList);
}
# SUGAR, REMOVE IN FUTURE
PROC tj_url2Collection(str collName, str doc, str url, str selTagList) : void
{
+ printf("WARNING: the use of function tj_url2Collection() is
deprecated.\n");
tj_set_parameter(collName,"tagFilter",selTagList);
_tj_add2collection(collName,_tj_collection(collName),url,doc,true);
}
@@ -2328,6 +2451,7 @@
# SUGAR, REMOVE IN FUTURE
PROC tj_url2Collection_index(str collName, str url, str selTagList) : void
{
+ printf("WARNING: the use of function tj_url2Collection_index() is
deprecated.\n");
tj_set_parameter(collName,"tagFilter",selTagList);
_tj_add2collection(collName,_tj_collection(collName),url,url,false);
}
@@ -2335,6 +2459,7 @@
# SUGAR, REMOVE IN FUTURE
PROC tj_finalize(str collName) : void
{
+ printf("WARNING: the use of function tj_finalize() is deprecated.\n");
}
PROC _buildIRindex(str collName) : void :=
-------------------------------------------------------------------------
Take Surveys. Earn Cash. Influence the Future of IT
Join SourceForge.net's Techsay panel and you'll get the chance to share your
opinions on IT & business topics through brief surveys-and earn cash
http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV
_______________________________________________
Monetdb-pf-checkins mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/monetdb-pf-checkins
