Update of /cvsroot/monetdb/pathfinder/compiler/mil
In directory sc8-pr-cvs16.sourceforge.net:/tmp/cvs-serv21943/compiler/mil
Modified Files:
mil.c milgen.brg milprint.c milprint_summer.c
Log Message:
This check-in contains the PTIJAH implementation using the Algebra version of
Pathfinder. All PFTIJAH functions are implemented using the new fun_call /
fun_param operations and (after some coaching by JanR:) this worked
pretty well.
Notes:
- all functions except tijah:ft-index-info() are implemented.
- The maintainance of the Full-Text indices is now also implemented using a
'tape'. So indices are only modified AFTER the query.
- Implementatation may be a little bit buggy because only operations on
small synthetic test examples have been test an no real complex scenarios
have been tried.
The coming weeks I will try to solve these small problems but I thought it
is better to synchronize early.
Jan Flokstra.
U milprint.c
Index: milprint.c
===================================================================
RCS file: /cvsroot/monetdb/pathfinder/compiler/mil/milprint.c,v
retrieving revision 1.68
retrieving revision 1.69
diff -u -d -r1.68 -r1.69
--- milprint.c 3 Apr 2008 09:20:43 -0000 1.68
+++ milprint.c 10 Apr 2008 13:38:43 -0000 1.69
@@ -355,6 +355,15 @@
, [m_catch] = "CATCH"
, [m_error] = "ERROR"
, [m_col_name] = "col_name"
+#ifdef HAVE_PFTIJAH
+ , [m_tj_pfop] = "ALG_tj_pfop"
+ , [m_tj_query_score] = "ALG_tj_query_score"
+ , [m_tj_query_nodes] = "ALG_tj_query_nodes"
+ , [m_tj_tokenize] = "[tijah_tokenize]"
+ , [m_tj_query_handler] = "ALG_tj_query_handler"
+ , [m_tj_add_fti_tape] = "ALG_tj_add_fti_tape"
+ , [m_tj_docmgmt_tape ] = "ALG_tj_docmgmt_tape"
+#endif
};
@@ -1086,6 +1095,91 @@
milprintf (")");
break;
+#ifdef HAVE_PFTIJAH
+ case m_tj_tokenize:
+ milprintf ("%s (", ID[n->kind]);
+ print_expression (n->child[0]);
+ milprintf (")");
+ break;
+ case m_tj_query_score:
+ milprintf ("%s (", ID[n->kind]);
+ print_expression (n->child[0]);
+ milprintf (", ");
+ print_expression (n->child[1]);
+ milprintf (", ");
+ print_expression (n->child[2]);
+ milprintf (", ");
+ print_expression (n->child[3]);
+ milprintf (")");
+ break;
+ case m_tj_query_nodes:
+ milprintf ("%s (", ID[n->kind]);
+ print_expression (n->child[0]);
+ milprintf (", ");
+ print_expression (n->child[1]);
+ milprintf (", ");
+ print_expression (n->child[2]);
+ milprintf (")");
+ break;
+ case m_tj_pfop:
+ milprintf ("%s (", ID[n->kind]);
+ print_expression (n->child[0]);
+ milprintf (", ");
+ print_expression (n->child[1]);
+ milprintf (", ");
+ print_expression (n->child[2]);
+ milprintf (", ");
+ print_expression (n->child[3]);
+ milprintf (")");
+ break;
+ case m_tj_query_handler:
+ milprintf ("%s (", ID[n->kind]);
+ print_expression (n->child[0]);
+ milprintf (", ");
+ print_expression (n->child[1]);
+ milprintf (", ");
+ print_expression (n->child[2]);
+ milprintf (", ");
+ print_expression (n->child[3]);
+ milprintf (", ");
+ print_expression (n->child[4]);
+ milprintf (", ");
+ print_expression (n->child[5]);
+ milprintf (", ");
+ print_expression (n->child[6]);
+ milprintf (")");
+ break;
+ case m_tj_add_fti_tape:
+ milprintf ("%s (", ID[n->kind]);
+ print_expression (n->child[0]);
+ milprintf (", ");
+ print_expression (n->child[1]);
+ milprintf (", ");
+ print_expression (n->child[2]);
+ milprintf (", ");
+ print_expression (n->child[3]);
+ milprintf (", ");
+ print_expression (n->child[4]);
+ milprintf (", ");
+ print_expression (n->child[5]);
+ milprintf (")");
+ break;
+ case m_tj_docmgmt_tape:
+ milprintf ("%s (", ID[n->kind]);
+ print_expression (n->child[0]);
+ milprintf (", ");
+ print_expression (n->child[1]);
+ milprintf (", ");
+ print_expression (n->child[2]);
+ milprintf (", ");
+ print_expression (n->child[3]);
+ milprintf (", ");
+ print_expression (n->child[4]);
+ milprintf (", ");
+ print_expression (n->child[5]);
+ milprintf (")");
+ break;
+#endif
default:
debug_output; /* Print MIL code so far when in debug mode. */
#ifndef NDEBUG
U mil.c
Index: mil.c
===================================================================
RCS file: /cvsroot/monetdb/pathfinder/compiler/mil/mil.c,v
retrieving revision 1.58
retrieving revision 1.59
diff -u -d -r1.58 -r1.59
--- mil.c 3 Apr 2008 09:20:41 -0000 1.58
+++ mil.c 10 Apr 2008 13:38:42 -0000 1.59
@@ -347,6 +347,11 @@
case PF_MIL_VAR_TRACE_TYPE: return "trace_type";
case PF_MIL_VAR_TRACE_REL: return "trace_rel";
+#ifdef HAVE_PFTIJAH
+ case PF_MIL_TIJAH_SCORE_DB: return "tijah_scoreDB";
+ case PF_MIL_TIJAH_FTI_TAPE: return "tijah_ftiTape";
+#endif
+
case PF_MIL_VAR_AXIS_ANC: return "AXIS_ancestor";
case PF_MIL_VAR_AXIS_ANC_S: return "AXIS_ancestor_or_self";
case PF_MIL_VAR_AXIS_CHLD: return "AXIS_child";
@@ -1668,4 +1673,52 @@
return wire1 (m_docmgmt_tape, args);
}
+#ifdef HAVE_PFTIJAH
+
+PFmil_t *
+PFmil_tj_tokenize (const PFmil_t *a)
+{
+ return wire1 (m_tj_tokenize, a);
+}
+
+/** pftijah algebra argument constructor */
+PFmil_t *
+PFmil_tj_pfop (const PFmil_t *a, const PFmil_t *b, const PFmil_t *c, const
PFmil_t *d)
+{
+ return wire4 (m_tj_pfop, a, b, c, d);
+}
+
+PFmil_t *
+PFmil_tj_query_score (const PFmil_t *a, const PFmil_t *b, const PFmil_t *c,
const PFmil_t *d)
+{
+ return wire4 (m_tj_query_score, a, b, c, d);
+}
+
+PFmil_t *
+PFmil_tj_query_nodes (const PFmil_t *a, const PFmil_t *b, const PFmil_t *c)
+{
+ return wire3 (m_tj_query_nodes, a, b, c);
+}
+
+/** pftijah main query handler */
+PFmil_t *
+PFmil_tj_query_handler (const PFmil_t *a, const PFmil_t *b, const PFmil_t *c,
const PFmil_t *d, const PFmil_t *e, const PFmil_t *f, const PFmil_t *g)
+{
+ return wire7 (m_tj_query_handler, a, b, c, d, e, f, g);
+}
+
+PFmil_t *
+PFmil_tj_add_fti_tape (const PFmil_t *a, const PFmil_t *b, const PFmil_t *c,
const PFmil_t *d, const PFmil_t *e, const PFmil_t *f)
+{
+ return wire6 (m_tj_add_fti_tape, a, b, c, d, e, f);
+}
+
+PFmil_t *
+PFmil_tj_docmgmt_tape (const PFmil_t *a, const PFmil_t *b, const PFmil_t *c,
const PFmil_t *d, const PFmil_t *e, const PFmil_t *f)
+{
+ return wire6 (m_tj_docmgmt_tape, a, b, c, d, e, f);
+}
+
+#endif
+
/* vim:set shiftwidth=4 expandtab: */
U milgen.brg
Index: milgen.brg
===================================================================
RCS file: /cvsroot/monetdb/pathfinder/compiler/mil/milgen.brg,v
retrieving revision 1.132
retrieving revision 1.133
diff -u -d -r1.132 -r1.133
--- milgen.brg 8 Apr 2008 12:16:49 -0000 1.132
+++ milgen.brg 10 Apr 2008 13:38:43 -0000 1.133
@@ -84,6 +84,7 @@
#include "pathfinder.h"
#include <assert.h>
+#include <string.h>
#include <stdio.h>
#include "oops.h"
@@ -603,6 +604,7 @@
#endif
{
mvar_t *mvar = env_mvar_unsafe (env, att, ty);
+
if (mvar)
return mvar;
@@ -2301,6 +2303,89 @@
} /* fold) */
/* fold) */
+#ifdef HAVE_PFTIJAH
+
+/* PFTIJAH helper functions */
+
+mvar_t
+*pft_create_pfop(PFpa_op_t *p, PFalg_simple_type_t t)
+{
+ mvar_t *res;
+ if ( p ) {
+ // res = new_var (p->refctr);
+ res = new_var (1);
+ mvar_t *p_iter = env_mvar (p->env, att_iter, aat_nat);
+ // fprintf(stderr,"p_iter = %d, pin =
%d.\n",p_iter->name-83,p_iter->pins);
+ mvar_t *p_item = env_mvar (p->env, att_item, t);
+ // fprintf(stderr,"p_item = %d, pin =
%d.\n",p_item->name-83,p_item->pins);
+ mvar_t *p_frag;
+ if ( t == aat_pre ) {
+ p_frag = env_mvar (p->env, att_item, aat_frag);
+ } else {
+ p_frag = new_var (1);
+ execute(
+ assgn (var (p_frag->name), lit_int(0))
+ );
+ }
+ // fprintf(stderr,"p_frag = %d, pin =
%d.\n",p_frag->name-83,p_frag->pins);
+ mvar_t *p_pos = env_mvar (p->env, att_pos , aat_nat);
+
+ if ( !(p_iter && p_item && p_frag && p_pos) )
+ fprintf(stderr,"# XXXXXXXXXXX [%d,%d,%d,%d] not
found.\n",(int)p_iter,(int)p_item,(int)p_frag,(int)p_pos);
+ execute (
+ assgn (var (res->name),
+ tj_pfop(var
(p_iter->name),var(p_item->name),var(p_frag->name),var(p_pos->name)))
+ );
+ if ( t != aat_pre )
+ unpin(p_frag,1);
+ } else {
+ res = new_var (1);
+ execute (
+ assgn (var (res->name),
+ seqbase (new (type (mty_void), type(mty_bat) ),
+ lit_oid (0)))
+ );
+ }
+ return res;
+}
+
+void
+pft_unpack_pfop(PFpa_op_t* dest, mvar_t *pack,
+ PFalg_att_t a_iter,
+ PFalg_att_t a_item, PFalg_simple_type_t t_item,
+ PFalg_att_t a_frag,
+ PFalg_att_t a_pos
+ )
+{
+ mvar_t *v_iter = new_var (dest->refctr);
+ execute (
+ assgn (var (v_iter->name),
+ fetch (var (pack->name), lit_int (0))));
+ env_add (dest->env, a_iter, aat_nat, v_iter);
+
+ mvar_t *v_item = new_var (dest->refctr);
+ execute (
+ assgn (var (v_item->name),
+ fetch (var (pack->name), lit_int (1))));
+ env_add (dest->env, a_item, t_item, v_item);
+
+ if ( a_frag ) {
+ mvar_t *v_frag = new_var (dest->refctr);
+ execute (
+ assgn (var (v_frag->name),
+ fetch (var (pack->name), lit_int (2))));
+ env_add (dest->env, a_frag, aat_frag, v_frag);
+ }
+
+ mvar_t *v_pos = new_var (dest->refctr);
+ execute (
+ assgn (var (v_pos->name),
+ fetch (var (pack->name), lit_int (3))));
+ env_add (dest->env, a_pos, aat_nat, v_pos);
+}
+
+#endif
+
/**
* @brief Reducer function.
*
@@ -2640,6 +2725,20 @@
arg (var (colname->name),
var (docmgmt->name)))))));
+#ifdef HAVE_PFTIJAH
+ // INCOMPLETE, when the print is removed it is optimized out
+ execute (
+ if_(
+ tj_docmgmt_tape (
+ var(PF_MIL_TIJAH_FTI_TAPE),
+ var (PF_MIL_VAR_WS),
+ var (filepath->name),
+ var (docname->name),
+ var (colname->name),
+ var (docmgmt->name)),
+ nil(),print(lit_str("INCOMPLETE fooled MIL
optimizer")))
+ );
+#endif
unpin (docmgmt, 1);
unpin (docname, 1);
unpin (colname, 1);
@@ -4329,6 +4428,7 @@
att3,
aat_dbl))))));
} break; /* fold) */
+
case alg_fun_fn_string_length: /* fold( */
{
PFalg_att_t att;
@@ -8121,6 +8221,8 @@
/* */
/***********************************/
+ // PFalg_att_t funcall_loop = p->sem.fun_call.iter;
+
if (p->sem.fun_call.kind == alg_fun_call_xrpc) {
PFcnode_t *core_apply = (PFcnode_t *) p->sem.fun_call.ctx;
@@ -8149,8 +8251,305 @@
env_add (p->env, p->schema.items[i].name, t, v);
}
}
+ } else if (p->sem.fun_call.kind == alg_fun_call_tijah) {
+#ifdef HAVE_PFTIJAH
- }
+/*
+ * PFTIJAH defines. I decided not make a seperate include file for
+ * pftijah so ensure that these defines are exactly the same as in
+ * ../algebra/builtins.c
+ */
+
+#define MYNODEKIND aat_pnode
+#define DOCMGMTTYPE aat_docmgmt
+
+#define PFT_FUN(F) (strncmp(F,"pftijah_",8)==0)
+
+#define PFT_QUERY_N_XX "pftijah_query_n_xx"
+#define PFT_QUERY_N_SX "pftijah_query_n_sx"
+#define PFT_QUERY_N_XO "pftijah_query_n_xo"
+#define PFT_QUERY_N_SO "pftijah_query_n_so"
+#define PFT_QUERY_I_XX "pftijah_query_i_xx"
+#define PFT_QUERY_I_SX "pftijah_query_i_sx"
+#define PFT_QUERY_I_XO "pftijah_query_i_xo"
+#define PFT_QUERY_I_SO "pftijah_query_i_so"
+
+#define PTF_QUERY_NODES(N) (N[14]=='n')
+#define PTF_QUERY_STARTNODES(N) (N[16]=='s')
+#define PTF_QUERY_OPTIONS(N) (N[17]=='o')
+
+#define PFT_FUN_QUERY(F) (strncmp(F,"pftijah_query_",14)==0)
+
+#define PFT_MANAGE_FTI_C_XX "pftijah_manage_fti_c_xx"
+#define PFT_MANAGE_FTI_C_CX "pftijah_manage_fti_c_cx"
+#define PFT_MANAGE_FTI_C_XO "pftijah_manage_fti_c_xo"
+#define PFT_MANAGE_FTI_C_CO "pftijah_manage_fti_c_co"
+#define PFT_MANAGE_FTI_E_CX "pftijah_manage_fti_e_cx"
+#define PFT_MANAGE_FTI_E_CO "pftijah_manage_fti_e_co"
+#define PFT_MANAGE_FTI_R_XX "pftijah_manage_fti_r_xx"
+#define PFT_MANAGE_FTI_R_XO "pftijah_manage_fti_r_xo"
+
+#define PFT_FUN_MANAGE(F) (strncmp(F,"pftijah_manage_",15)==0)
+#define PFT_FUN_MANAGE_KIND(F) (F[19])
+#define PFT_FUN_MANAGE_COLL(F) (F[21] == 'c')
+#define PFT_FUN_MANAGE_OPT(F) (F[22] == 'o')
+
+#define PFT_SCORE "pftijah_score"
+#define PFT_NODES "pftijah_nodes"
+#define PFT_INFO "pftijah_info"
+#define PFT_TOKENIZE "pftijah_tokenize"
+#define PFT_RESSIZE "pftijah_ressize"
+
+ const char* fcname = PFqname_loc(p->sem.fun_call.qname);
+
+ PFalg_att_t fc_iter_a = p->schema.items[0].name; /* fixed */
+ PFalg_att_t fc_pos_a = p->schema.items[1].name; /* fixed */
+ PFalg_att_t fc_item_a = p->schema.items[2].name; /* fixed */
+
+
+ if ( PFT_FUN(fcname) ) {
+
+ /* handle pftijah functions here */
+
+#ifdef FUNCALL_DEBUG
+ fprintf(stderr,"# PFTIJAH fun_call() HANDLER START\n");
+ fprintf(stderr,"# Called function is: %s\n",fcname);
+ fprintf(stderr,"# number of args is:
%d\n",PFarray_last(fun_params));
+
+ fprintf(stderr,"#FUNCALL/SCHEMA:\n");
+ for (unsigned int qq=0; qq < p->schema.count; qq++) {
+ struct PFalg_schm_item_t s_item = p->schema.items[qq];
+
fprintf(stderr,"#.ITEM[%d]=(%s:%d)\n",qq,PFatt_str(s_item.name),s_item.type);
+
+ }
+ for (unsigned int ii=0; ii<PFarray_last(fun_params); ii++) {
+ fprintf(stderr,"#FUNCALL/ENV[%d]\n",ii);
+ const PFarray_t *env = (*(PFpa_op_t **)PFarray_at
(fun_params,ii))->env;
+ for (unsigned int iii = 0; iii < PFarray_last (env);
iii++) {
+
+ env_t entry = *(env_t *) PFarray_at ((PFarray_t *) env,
iii);
+ fprintf(stderr,"#.BAT[a%d]
pin=%d\n",entry.mvar->name-83,entry.mvar->pins);
+ }
+ }
+#endif /* FUNCALL_DEBUG */
+
+ mvar_t *fc_loop = env_mvar (L(p)->env, p->sem.fun_call.iter ,
aat_nat);
+
+ if ( PFT_FUN_QUERY(fcname) ) {
+ int fcci = 0;
+ PFalg_simple_type_t restype;
+ int returnNodes = PTF_QUERY_NODES(fcname);
+
+ mvar_t *retNodes = new_var (1);
+ if ( returnNodes ) {
+ execute (assgn(var(retNodes->name),lit_bit(false)));
+ restype = aat_pre;
+ } else {
+ execute (assgn(var(retNodes->name),lit_bit(true)));
+ restype = aat_int;
+ }
+
+ mvar_t *pfop_sn, *pfop_query, *pfop_opt;
+
+ if ( PTF_QUERY_STARTNODES(fcname) ) {
+ pfop_sn = pft_create_pfop(*(PFpa_op_t **)PFarray_at
(fun_params,fcci),aat_pre);
+ fcci++;
+ } else {
+ pfop_sn = pft_create_pfop(NULL,aat_pre);
+ }
+
+ pfop_query = pft_create_pfop(*(PFpa_op_t **)PFarray_at
(fun_params,fcci),aat_str);
+ fcci++;
+
+ if ( PTF_QUERY_OPTIONS(fcname) ) {
+ pfop_opt = pft_create_pfop(*(PFpa_op_t **)PFarray_at
(fun_params,fcci),aat_pre);
+ } else {
+ pfop_opt = pft_create_pfop(NULL,aat_pre);
+ }
+
+ mvar_t *tjqres = new_var(1);
+ execute(
+ assgn(var(tjqres->name),
+ tj_query_handler(
+ var(retNodes->name),
+ var(pfop_sn->name),
+ var(pfop_query->name),
+ var(pfop_opt->name),
+ var(fc_loop->name),
+ var (PF_MIL_VAR_WS),
+ var (PF_MIL_TIJAH_SCORE_DB)
+ )
+ )
+ );
+ unpin(retNodes,1);
+
+ unpin(pfop_sn,1);
+ unpin(pfop_query,1);
+ unpin(pfop_opt,1);
+
+
pft_unpack_pfop(p,tjqres,fc_iter_a,fc_item_a,restype,fc_item_a,fc_pos_a);
+
+ unpin(tjqres,1);
+ } else if ( PFT_FUN_MANAGE(fcname) ) {
+ int fcci = 0;
+
+ char* opkind;
+
+ switch (PFT_FUN_MANAGE_KIND(fcname) ) {
+ case 'c': opkind = "create";
+ break;
+ case 'e': opkind = "extend";
+ break;
+ case 'r': opkind = "remove";
+ break;
+ default: opkind = "error";
+ }
+
+ mvar_t *pfop_coll, *pfop_opt;
+
+ if ( PFT_FUN_MANAGE_COLL(fcname) ) {
+ pfop_coll = pft_create_pfop(*(PFpa_op_t
**)PFarray_at (fun_params,fcci),aat_str);
+ fcci++;
+ } else {
+ pfop_coll = pft_create_pfop(NULL,aat_pre);
+ }
+ if ( PFT_FUN_MANAGE_OPT(fcname) ) {
+ pfop_opt = pft_create_pfop(*(PFpa_op_t **)PFarray_at
(fun_params,fcci),aat_pre);
+ fcci++;
+ } else {
+ pfop_opt = pft_create_pfop(NULL,aat_pre);
+ }
+
+ execute (
+ assgn (var (PF_MIL_TIJAH_FTI_TAPE),
+ tj_add_fti_tape(
+ lit_str(opkind),
+ var(pfop_coll->name),
+ var(pfop_opt->name),
+ var(fc_loop->name),
+ var(PF_MIL_VAR_WS),
+ var(PF_MIL_TIJAH_FTI_TAPE)
+ )));
+
+ unpin(pfop_coll,1);
+ unpin(pfop_opt,1);
+
+ mvar_t *p_iter = new_var (p->refctr);
+ execute (
+ assgn (var (p_iter->name),
+ seqbase (new (type (mty_void), type(mty_oid) ),
+ lit_oid (0))));
+ env_add (p->env, fc_iter_a, aat_nat, p_iter);
+
+ mvar_t *p_item1 = new_var (p->refctr);
+ execute (
+ assgn (var (p_item1->name),
+ seqbase (new (type (mty_void), type(mty_lng) ),
+ lit_oid (0))));
+ env_add (p->env, fc_item_a, aat_docmgmt, p_item1);
+
+ mvar_t *p_item2 = new_var (p->refctr);
+ execute (
+ assgn (var (p_item2->name),
+ seqbase (new (type (mty_void), type(mty_str) ),
+ lit_oid (0))));
+ env_add (p->env, fc_item_a, aat_docnm, p_item2);
+
+ mvar_t *p_pos = new_var (p->refctr);
+ execute (
+ assgn (var (p_pos->name),
+ seqbase (new (type (mty_void), type(mty_oid) ),
+ lit_oid (0))));
+ env_add (p->env, fc_pos_a, aat_nat, p_pos);
+ } else if (strcmp(fcname,PFT_SCORE) == 0 ) {
+ mvar_t *pfop_id;
+ mvar_t *pfop_nodes;
+
+ pfop_id = pft_create_pfop(*(PFpa_op_t **)PFarray_at
(fun_params,0),aat_int);
+ pfop_nodes = pft_create_pfop(*(PFpa_op_t **)PFarray_at
(fun_params,1),aat_pre);
+
+ mvar_t *tjsres = new_var(1);
+ execute(
+ assgn(var(tjsres->name),
+ tj_query_score(
+ var(fc_loop->name),
+ var(pfop_id->name),
+ var(pfop_nodes->name),
+ var (PF_MIL_TIJAH_SCORE_DB)
+ )
+ )
+ );
+ unpin(pfop_id,1);
+ unpin(pfop_nodes,1);
+
+
pft_unpack_pfop(p,tjsres,fc_iter_a,fc_item_a,aat_dbl,0,fc_pos_a);
+
+ unpin(tjsres,1);
+ } else if (strcmp(fcname,PFT_NODES) == 0 ) {
+ mvar_t *pfop_id;
+
+ pfop_id = pft_create_pfop(*(PFpa_op_t **)PFarray_at
(fun_params,0),aat_int);
+
+ mvar_t *tjnres = new_var(1);
+ execute(
+ assgn(var(tjnres->name),
+ tj_query_nodes(
+ var(fc_loop->name),
+ var(pfop_id->name),
+ var (PF_MIL_TIJAH_SCORE_DB)
+ )
+ )
+ );
+ unpin(pfop_id,1);
+
+
pft_unpack_pfop(p,tjnres,fc_iter_a,fc_item_a,aat_pre,fc_item_a,fc_pos_a);
+
+ unpin(tjnres,1);
+ } else if (strcmp(fcname,PFT_INFO) == 0 ) {
+ PFoops (OOPS_FATAL,"milgen.brg: tijah:ft-index-info ()
temporary not supported.\n");
+ } else if ( (strcmp(fcname,PFT_TOKENIZE) == 0) ||
+ (strcmp(fcname,PFT_RESSIZE) == 0) ) {
+ PFpa_op_t* op1 = *(PFpa_op_t **)PFarray_at(fun_params,0);
+
+ // copy the iter bat
+ mvar_t *p_iter = env_mvar (op1->env, att_iter, aat_nat);
+ pin(p_iter,1);
+ env_add (p->env, fc_iter_a, aat_nat, p_iter);
+
+ mvar_t *new_item = new_var (p->refctr);
+ if ( strcmp(fcname,PFT_TOKENIZE) == 0 ) {
+ mvar_t *p_item = env_mvar (op1->env, att_item, aat_str);
+ execute (
+ assgn (var (new_item->name),
+ tj_tokenize( var(p_item->name) ) ) );
+ env_add (p->env, fc_item_a, aat_str, new_item);
+ } else {
+ // must be RESSIZE
+ mvar_t *p_item = env_mvar (op1->env, att_item, aat_int);
+ // "var res := item%s.join(tijah_scoreDB.fetch([EMAIL
PROTECTED]));\n"
+ execute (
+ assgn (var (new_item->name),
+ join( var(p_item->name),
+ fetch( var(PF_MIL_TIJAH_SCORE_DB),
+ lit_int(4)) ) ) );
+ env_add (p->env, fc_item_a, aat_int, new_item);
+ }
+
+ // copy the pos bat
+ mvar_t *p_pos = env_mvar (op1->env, att_iter, aat_nat);
+ pin(p_pos,1);
+ env_add (p->env, fc_pos_a, aat_nat, p_pos);
+ } else {
+ PFoops (OOPS_FATAL,"milgen.brg: PFTIJAH fun_call() bad
function[%s]!!!\n",fcname);
+ }
+ } else {
+ PFoops (OOPS_FATAL,"milgen.brg: PFTIJAH fun_call() bad
function[%s]!!!\n",fcname);
+ }
+#endif /* PFTIJAH */
+
+ } else {
+ // ERROR? do not know how to handle
+ }
/* Unpin all MIL variables that represent the function parameters.
(They have 1 additional pin (see rule fun_param) to ensure
@@ -8160,7 +8559,7 @@
for (unsigned int j = 0; j < PFarray_last (param->env); j++)
unpin (((env_t *) PFarray_at (param->env, j))->mvar, 1);
}
-
+
/* restore the function arguments */
fun_params = old_fun_params;
} break; /* fold) */
@@ -8186,7 +8585,6 @@
/* go on to next arguments */
reduce (kids[1], nts[1]);
} break; /* fold) */
-
/* Param: nil */
case 154:
/* end of parameter list */
@@ -8385,6 +8783,30 @@
milprog = seq (module (lit_str ("pathfinder")),
module (lit_str ("mkey")),
+#ifdef HAVE_PFTIJAH
+ /* create pftijah score db */
+ declare (var (PF_MIL_TIJAH_SCORE_DB)),
+ assgn (var (PF_MIL_TIJAH_SCORE_DB),
+ seqbase (new (type (mty_void), type(mty_bat) ),
+ lit_oid (0))),
+ append (var (PF_MIL_TIJAH_SCORE_DB),
+ seqbase (new (type (mty_void), type(mty_oid) ),
+ lit_oid (0))),
+ append (var (PF_MIL_TIJAH_SCORE_DB),
+ seqbase (new (type (mty_void), type(mty_oid) ),
+ lit_oid (0))),
+ append (var (PF_MIL_TIJAH_SCORE_DB),
+ seqbase (new (type (mty_void), type(mty_oid) ),
+ lit_oid (0))),
+ append (var (PF_MIL_TIJAH_SCORE_DB),
+ seqbase (new (type (mty_void), type(mty_dbl) ),
+ lit_oid (0))),
+ append (var (PF_MIL_TIJAH_SCORE_DB),
+ new (type (mty_lng), type(mty_lng) ) ),
+ declare (var (PF_MIL_TIJAH_FTI_TAPE)),
+ assgn (var (PF_MIL_TIJAH_FTI_TAPE),
+ (new (type (mty_str), type(mty_bat) ))),
+#endif
/* add timing information */
declare (var (PF_MIL_VAR_TIME_LOAD)),
declare (var (PF_MIL_VAR_TIME_QUERY)),
U milprint_summer.c
Index: milprint_summer.c
===================================================================
RCS file: /cvsroot/monetdb/pathfinder/compiler/mil/milprint_summer.c,v
retrieving revision 1.417
retrieving revision 1.418
diff -u -d -r1.417 -r1.418
--- milprint_summer.c 7 Apr 2008 12:48:10 -0000 1.417
+++ milprint_summer.c 10 Apr 2008 13:38:43 -0000 1.418
@@ -3648,8 +3648,6 @@
milprintf(f, "} # end of '%s' calculation\n", operator);
}
-#ifdef HAVE_PROBXML
-
static int
translateDeepEq (opt_t *f, int cur_level, int counter, char *comp, PFcnode_t
*args)
{
@@ -3662,7 +3660,8 @@
translate2MIL (f, VALUES, cur_level, counter, RL(args));
milprintf(f, "{ # pxmlsup:deep-equal calculation\n");
- milprintf(f, "var val_fst :=
probxml_deep_eq(ws,item%03u,kind%03u,item,kind);\n", counter,counter);
+ // INCOMPLETE implement loop-lifted version
+ milprintf(f, "var val_fst :=
fn_deep_eq(ws,item%03u,constant2bat(kind%03u),item,constant2bat(kind));\n",
counter,counter);
milprintf(f, "item := val_fst.[oid]();\n");
milprintf(f, "kind := BOOL;\n");
milprintf(f, "} # end of pxmlsup:deep-equal\n");
@@ -3672,7 +3671,6 @@
return NORMAL;
}
-#endif /*HAVE_PROBXML */
/**
* evaluateOpOpt evaluates a operation and gives back a new
@@ -8375,9 +8373,11 @@
ctx_counter = counter;
/* get query string */
- rc = translate2MIL (f, VALUES, cur_level, counter, RL(args));
+ rc = translate2MIL (f, VALUES, cur_level, counter, RL(args));
+#ifdef REMOVE
if (rc == NORMAL)
milprintf(f, "item%s := item.leftfetchjoin(str_values);\n",
item_ext);
+#endif
add_empty_strings (f, STR, cur_level);
saveResult_ (f, ++counter, STR);
str_counter = counter;
@@ -8413,157 +8413,62 @@
} else {
rc = translate2MIL (f, VALUES, cur_level, counter, RL(args));
}
+#ifdef REMOVE
if (rc == NORMAL)
milprintf(f, "item%s := item.leftfetchjoin(str_values);\n",
item_ext);
+#endif
add_empty_strings (f, STR, cur_level);
saveResult_ (f, ++counter, STR);
str_counter = counter;
} else if (fun->arity == 1) {
/* get query string */
- rc = translate2MIL (f, VALUES, cur_level, counter, L(args));
+ rc = translate2MIL (f, VALUES, cur_level, counter, L(args));
+#ifdef REMOVE
if (rc == NORMAL)
milprintf(f, "item%s := item.leftfetchjoin(str_values);\n",
item_ext);
+#endif
add_empty_strings (f, STR, cur_level);
saveResult_ (f, ++counter, STR);
- str_counter = counter;
- }
- if ( storeScore )
- milprintf(f,
- "var result_id := new(void,lng).seqbase([EMAIL PROTECTED]);");
- else
- milprintf(f,
- "var result_iter := new(void,oid).seqbase([EMAIL PROTECTED]);"
- "var result_item := new(void,oid).seqbase([EMAIL PROTECTED]);"
- "var result_pos := new(void,oid).seqbase([EMAIL PROTECTED]);"
- "var result_frag := new(void,oid).seqbase([EMAIL
PROTECTED]);");
-
- /* generate the serialization code */
- milprintf(f,
- "[EMAIL PROTECTED]() { # begin batloop over queries\n"
- , cur_level);
- if (opt_counter)
- milprintf(f,
- " iter := iter%03u.select($t);\n"
- " item := item%03u.materialize(ipik%03u).semijoin(iter);\n"
- " kind := kind%03u.materialize(ipik%03u).semijoin(iter);\n"
- " iter := iter.tmark([EMAIL PROTECTED]);\n"
- " item := item.tmark([EMAIL PROTECTED]);\n"
- " kind := kind.tmark([EMAIL PROTECTED]);\n"
- " var optbat :=
serialize_tijah_opt(ws,1,iter,iter,item,kind,int_values,dbl_values,str_values);\n"
- ,opt_counter,opt_counter, opt_counter,opt_counter,opt_counter);
- else
- milprintf(f,
- " var optbat := new(str,str,32);\n");
-
- milprintf(f,
- " var ftindex := tj_get_ft_index(optbat,true);\n"
- " tijah_lock := tj_get_collection_lock(ftindex);\n"
- " lock_set(tijah_lock);\n"
- );
- if ( ctx_counter ) {
- milprintf(f,
- " var startNodes;\n"
- " iter := iter%03u.materialize(ipik%03u);\n"
- " var iteration := iter%03u.fetch(int($h));\n"
- " iter := iter.select(iteration);\n"
- " iteration := nil;\n"
- " item := item%03u.materialize(ipik%03u).semijoin(iter);\n"
- " kind := kind%03u.materialize(ipik%03u).semijoin(iter);\n"
- " iter := iter.tmark([EMAIL PROTECTED]);\n"
- " item := item.tmark([EMAIL PROTECTED]);\n"
- " kind := kind.tmark([EMAIL PROTECTED]);\n"
- " var xdoc_name := bat(\"tj_\" + ftindex +
\"_doc_name\");\n"
- " var xdoc_firstpre := bat(\"tj_\" + ftindex +
\"_doc_firstpre\");\n"
- " var xpfpre := bat(\"tj_\" + ftindex + \"_pfpre\");\n"
- " var doc_loaded :=
reverse(ws.fetch(OPEN_CONT)).leftfetchjoin(ws.fetch(OPEN_NAME));\n"
- " startNodes :=
pf2tijah_node(xdoc_name,xdoc_firstpre,xpfpre,item,kind,doc_loaded);\n"
- , ctx_counter, ctx_counter, str_counter, ctx_counter,
ctx_counter, ctx_counter, ctx_counter);
- } else {
- milprintf(f," var startNodes := new(void,oid);\n");
- }
- /* execute tijah query */
- milprintf(f,
- " var nexi_allscores :=
run_tijah_query(ftindex,optbat,%s,startNodes,item%s%03u.fetch(int($h)));\n"
- " var nexi_score;\n"
- " if ( optbat.exist(\"returnNumber\") ) {\n"
- " var retNum := int(optbat.find(\"returnNumber\"));\n"
- " nexi_score := nexi_allscores.slice(0, retNum - 1);\n"
- " } else {\n"
- " nexi_score := nexi_allscores;\n"
- " }\n"
- , (ctx_counter?"true":"false"),item_ext, str_counter);
- /* translate tijah_pre to pf-pre */
- milprintf(f,
- " var docpre := bat(\"tj_\" + ftindex +
\"_doc_firstpre\").[oid]();\n"
- " var pfpre := bat(\"tj_\" + ftindex + \"_pfpre\");\n"
- " item := nexi_score.hmark([EMAIL PROTECTED]);\n"
- " var frag := [find_lower](const
docpre.reverse().mark([EMAIL PROTECTED]), item);\n"
- " item := item.join(pfpre).sort().tmark();\n"
- " var needed_docs := bat(\"tj_\" + ftindex +
\"_doc_name\").semijoin(frag.tunique());\n"
- " lock_unset(tijah_lock); tijah_lock := lock_nil;\n"
- " var loaded_docs := ws.fetch(OPEN_NAME).reverse();\n"
- " var docs_to_load :=
kdiff(needed_docs.reverse(),loaded_docs).hmark([EMAIL PROTECTED]);\n"
- " ws_opendoc(ws, docs_to_load);\n"
- " docs_to_load := nil;\n"
- " loaded_docs := nil;\n"
- " var doc_loaded :=
reverse(ws.fetch(OPEN_CONT)).leftfetchjoin(ws.fetch(OPEN_NAME));\n"
- " var fid_pffid :=
needed_docs.join(doc_loaded.reverse());\n"
- " frag := frag.join(fid_pffid).sort().tmark();\n");
-
- if ( storeScore ) {
- /* store scores and nodes */
- milprintf(f,
- " tID := oid(int(tID) + 1);\n"
- "
tijah_resultsz.insert(lng(tID),lng(nexi_allscores.count()));\n"
- " tijah_tID.append(item.project(tID));\n"
- " tijah_frag.append(frag);\n"
- " tijah_pre.append(item);\n"
- " tijah_score.append(nexi_score.tmark());\n"
- " result_id.append(lng(tID));\n"
- "} # end batloop over queries\n");
-
- /* return query identifier */
- item_ext = (code)?kind_str(INT):"";
- if (code)
- milprintf(f, "item%s := result_id;\n", item_ext);
- else
- addValues (f, int_container(), "result_id", "item");
-
- milprintf(f,
- "iter := loop%03u.tmark(oid(0));\n"
- "ipik := iter;\n"
- "pos := oid(1);\n"
- "kind := INT;\n"
- , cur_level);
- } else {
- /* do not store score, return nodes instead */
- milprintf(f,
- " result_iter.append(item.project($t));\n"
- " result_pos.append(item.mark([EMAIL PROTECTED]));\n"
- " result_frag.append(frag);\n"
- " result_item.append(item);\n"
- "} # end batloop over queries\n");
- milprintf(f,
- "iter := result_iter;\n"
- "result_iter := nil;\n"
- "pos := result_pos;\n"
- "result_pos := nil;\n"
- "kind := set_kind(result_frag, ELEM);\n"
- "result_frag := nil;\n"
- "item := result_item;\n"
- "result_item := nil;\n"
- "ipik := iter;\n"
- );
- }
-
- /* clean up */
- if ( opt_counter )
- deleteResult(f, opt_counter);
+ str_counter = counter;
+ }
+ /* PARAMETER COMPUTATION */
+ milprintf(f,
+ "var pfop_query :=
tj_pfop(iter%03u.materialize(ipik%03u),item%s%03u.materialize(ipik%03u),kind%03u.materialize(ipik%03u),pos%03u.materialize(ipik%03u));\n"
+
,str_counter,str_counter,item_ext,str_counter,str_counter,str_counter,str_counter,str_counter,str_counter);
+ if ( opt_counter )
+ milprintf(f,
+ "var pfop_opt :=
tj_pfop(iter%03u.materialize(ipik%03u),item%03u.materialize(ipik%03u),kind%03u.materialize(ipik%03u),pos%03u.materialize(ipik%03u));\n"
+
,opt_counter,opt_counter,opt_counter,opt_counter,opt_counter,opt_counter,opt_counter,opt_counter);
+ else
+ milprintf(f,
+ "var pfop_opt := new(void,bat);\n");
+ if ( ctx_counter )
+ milprintf(f,
+ "var pfop_sn :=
tj_pfop(iter%03u.materialize(ipik%03u),item%03u.materialize(ipik%03u),kind%03u.materialize(ipik%03u),pos%03u.materialize(ipik%03u));\n"
+
,ctx_counter,ctx_counter,ctx_counter,ctx_counter,ctx_counter,ctx_counter,ctx_counter,ctx_counter);
+ else
+ milprintf(f,
+ "var pfop_sn := new(void,bat);\n");
+ milprintf(f,
+ "# call the tijah query handler\n"
+ " var res := tj_query_handler(%s, pfop_sn, pfop_query,
pfop_opt, loop%03u, ws, int_values, dbl_values, str_values, tijah_scoreDB);\n"
+ ,(storeScore?"TRUE":"FALSE"),cur_level);
+ milprintf(f,
+ "# unpack the returned result\n"
+ "iter := res.fetch([EMAIL PROTECTED]);\n"
+ "item := res.fetch([EMAIL PROTECTED]);\n"
+ "kind := res.fetch([EMAIL PROTECTED]);\n"
+ "pos := res.fetch([EMAIL PROTECTED]);\n"
+ );
+ /* END OF SCRIPT */
+ /* clean up */
+ if ( opt_counter )
+ deleteResult(f, opt_counter);
deleteResult_ (f, str_counter, STR);
if ( ctx_counter )
deleteResult(f, ctx_counter);
milprintf(f, "} # end of translate tijah:query[all][-id]\n");
- return (code && storeScore)?INT:NORMAL;
+ return NORMAL;
}
else if ( !PFqname_eq(fnQname, PFqname (PFns_tijah,"nodes")) )
{
@@ -8587,15 +8492,15 @@
"var frag := new(void,oid).seqbase([EMAIL PROTECTED]);"
"[EMAIL PROTECTED]() { # begin of query batloop\n"
" var qid := oid(item%s.fetch(int($h)));\n"
- " var tmp := tijah_tID.ord_uselect(qid);\n"
- " item.append(tmp.mirror().leftfetchjoin(tijah_pre));\n"
- " iter.append(tmp.project(loop%03u.fetch(int($h))));\n"
- " frag.append(tmp.mirror().leftfetchjoin(tijah_frag));\n"
- " pos.append(tmp.mark([EMAIL PROTECTED]));\n"
- "} # end of query batloop \n"
- , cur_level, item_int, cur_level);
-
- milprintf(f,
+ " var tmp := tijah_scoreDB.fetch([EMAIL
PROTECTED]).ord_uselect(qid);\n"
+ "
item.append(tmp.mirror().leftfetchjoin(tijah_scoreDB.fetch([EMAIL
PROTECTED])));\n"
+ " iter.append(tmp.project(loop%03u.fetch(int($h))));\n"
+ "
frag.append(tmp.mirror().leftfetchjoin(tijah_scoreDB.fetch([EMAIL
PROTECTED])));\n"
+ " pos.append(tmp.mark([EMAIL PROTECTED]));\n"
+ "} # end of query batloop \n"
+ , cur_level, item_int, cur_level);
+
+ milprintf(f,
"kind := set_kind(frag, ELEM);\n"
"ipik := iter;\n"
"} # end of translate pf:tijah_nodes\n");
@@ -8618,35 +8523,35 @@
/* get node */
rc = translate2MIL (f, code, cur_level, counter, RL(args));
-
- /* get scores */
- milprintf(f,
- "var score := new(oid,dbl);\n"
- "var tmp := [<<]([lng](tijah_frag), const 32);\n"
- "var tijah_fragpre := [+](tmp, [lng](tijah_pre));\n"
- "tmp := nil;\n"
- "var item1_unique := item%s%03u.tunique();\n"
- "item := item.materialize(ipik);"
- "kind := kind.materialize(ipik);"
+
+ /* get scores */
+ milprintf(f,
+ "var score := new(oid,dbl);\n"
+ "var tmp := [<<]([lng](tijah_scoreDB.fetch([EMAIL PROTECTED])),
const 32);\n"
+ "var tijah_fragpre := [+](tmp, [lng](tijah_scoreDB.fetch([EMAIL
PROTECTED])));\n"
+ "tmp := nil;\n"
+ "var item1_unique := item%s%03u.tunique();\n"
+ "item := item.materialize(ipik);"
+ "kind := kind.materialize(ipik);"
"[EMAIL PROTECTED]() { # begin query batloop\n"
- " var item_part := item.semijoin(item%s%03u.uselect($h));\n"
- " var frag_part :=
kind.semijoin(item_part).get_container();\n"
- " frag_part := [<<]([lng](frag_part), const 32);\n"
- " var fragpre_part := [+](frag_part, [lng](item_part));\n"
- " item_part := nil;\n"
- " frag_part := nil;\n"
- " tmp := tijah_tID.uselect(oid($h));\n"
- " tmp := tmp.mirror().leftfetchjoin(tijah_fragpre);\n"
- " tmp := tmp.join(fragpre_part.reverse());\n"
- " score.insert(tmp.reverse().leftfetchjoin(tijah_score));\n"
- "} # end query batloop\n"
- "var xitem := kdiff(item,score).project(dbl(0));\n"
- "score.insert(xitem);\n"
- "xitem := nil;\n"
- "score := score.sort().tmark([EMAIL PROTECTED]);\n"
- , item_int, counter, item_int, counter);
-
- /* return score */
+ " var item_part := item.semijoin(item%s%03u.uselect($h));\n"
+ " var frag_part :=
kind.semijoin(item_part).get_container();\n"
+ " frag_part := [<<]([lng](frag_part), const 32);\n"
+ " var fragpre_part := [+](frag_part, [lng](item_part));\n"
+ " item_part := nil;\n"
+ " frag_part := nil;\n"
+ " tmp := tijah_scoreDB.fetch([EMAIL
PROTECTED]).uselect(oid($h));\n"
+ " tmp := tmp.mirror().leftfetchjoin(tijah_fragpre);\n"
+ " tmp := tmp.join(fragpre_part.reverse());\n"
+ "
score.insert(tmp.reverse().leftfetchjoin(tijah_scoreDB.fetch([EMAIL
PROTECTED])));\n"
+ "} # end query batloop\n"
+ "var xitem := kdiff(item,score).project(dbl(0));\n"
+ "score.insert(xitem);\n"
+ "xitem := nil;\n"
+ "score := score.sort().tmark([EMAIL PROTECTED]);\n"
+ , item_int, counter, item_int, counter);
+
+ /* return score */
item_ext = (code)?kind_str(DBL):"";
if (code)
milprintf(f, "item%s := score;\n", item_ext);
@@ -8719,8 +8624,8 @@
milprintf(f,
"if (ipik.count() != 0) { # tijah:resultsize\n"
- "var res := item%s.join(tijah_resultsz);\n"
- ,item_ext);
+ "var res := item%s.join(tijah_scoreDB.fetch([EMAIL
PROTECTED]));\n"
+ ,item_ext);
if (code)
milprintf(f, "item%s := res;\n", item_ext);
else
@@ -8740,7 +8645,8 @@
translateIntersect (f, "diff", cur_level, counter, args);
return NORMAL;
}
- else if (!PFqname_eq(fnQname,PFqname (PFns_pxmlsup,"deep-equal")))
+ else if (!PFqname_eq(fnQname,PFqname (PFns_pxmlsup,"deep-equal")) ||
+ !PFqname_eq(fnQname,PFqname (PFns_fn,"deep-equal")) )
{
return translateDeepEq (f, cur_level, counter, "deep-equal", args);
}
@@ -11534,13 +11440,13 @@
const char* PFinitMIL(void) {
return
#ifdef HAVE_PFTIJAH
- "var tID := [EMAIL PROTECTED]; # start counter at an arbitrary
number\n"
- "var tijah_tID := new(void,oid).seqbase([EMAIL PROTECTED]);\n"
- "var tijah_frag := new(void,oid).seqbase([EMAIL PROTECTED]);\n"
- "var tijah_pre := new(void,oid).seqbase([EMAIL PROTECTED]);\n"
- "var tijah_score := new(void,dbl).seqbase([EMAIL PROTECTED]);\n"
- "var tijah_resultsz := new(lng,lng);\n"
- "var tijah_lock := lock_nil; # pftijah collection lock\n"
+ "var tijah_scoreDB := new(void,bat).seqbase([EMAIL PROTECTED]);\n"
+ "tijah_scoreDB.append(new(void,oid).seqbase([EMAIL PROTECTED]));\n" //
[EMAIL PROTECTED] tijah_tID
+ "tijah_scoreDB.append(new(void,oid).seqbase([EMAIL PROTECTED]));\n" //
[EMAIL PROTECTED] tijah_frag
+ "tijah_scoreDB.append(new(void,oid).seqbase([EMAIL PROTECTED]));\n" //
[EMAIL PROTECTED] tijah_pre
+ "tijah_scoreDB.append(new(void,dbl).seqbase([EMAIL PROTECTED]));\n" //
[EMAIL PROTECTED] tijah_score
+ "tijah_scoreDB.append(new(lng,lng));\n" // [EMAIL
PROTECTED] tijah_resultsz
+ "var tijah_lock := lock_nil; # pftijah collection lock\n"
#endif
#ifdef HAVE_PROBXML
"module(\"probxml\");\n"
-------------------------------------------------------------------------
This SF.net email is sponsored by the 2008 JavaOne(SM) Conference
Don't miss this year's exciting event. There's still time to save $100.
Use priority code J8TL2D2.
http://ad.doubleclick.net/clk;198757673;13503038;p?http://java.sun.com/javaone
_______________________________________________
Monetdb-pf-checkins mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/monetdb-pf-checkins
