Changeset: 5829c78735c0 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5829c78735c0 Added Files: sql/test/merge-partitions/Tests/mergepart09.sql Modified Files: monetdb5/modules/mal/wlc.mal sql/backends/monet5/sql_cat.c sql/backends/monet5/sqlcatalog.mal sql/backends/monet5/wlr.mal sql/common/sql_list.c sql/include/sql_catalog.h sql/server/rel_optimizer.c sql/server/rel_schema.c sql/storage/sql_catalog.c sql/storage/sql_storage.h sql/storage/store.c sql/test/merge-partitions/Tests/All Branch: merge-partitions Log Message:
Propagate inserts in value partitioned tables. Had to replace the SQL definition from a BAT of values to a SQL list of values to be used in rel optimizer. diffs (truncated from 736 to 300 lines): diff --git a/monetdb5/modules/mal/wlc.mal b/monetdb5/modules/mal/wlc.mal --- a/monetdb5/modules/mal/wlc.mal +++ b/monetdb5/modules/mal/wlc.mal @@ -246,6 +246,10 @@ unsafe pattern transaction() address WLCgeneric comment "Start an autocommit transaction"; -pattern alter_add_value_partition( sname:str, mtnme:str, psnme:str, ptnme:str, arg:str...) +pattern alter_add_value_partition( sname:str, mtnme:str, psnme:str, ptnme:str, nills:int) address WLCgeneric comment "Catalog operation alter_add_value_partition"; + +pattern alter_add_value_partition( sname:str, mtnme:str, psnme:str, ptnme:str, nills:int, arg:str...) +address WLCgeneric +comment "Catalog operation alter_add_value_partition"; diff --git a/sql/backends/monet5/sql_cat.c b/sql/backends/monet5/sql_cat.c --- a/sql/backends/monet5/sql_cat.c +++ b/sql/backends/monet5/sql_cat.c @@ -302,7 +302,7 @@ alter_table_add_range_partition(mvc *sql break; case -4: assert(err); - if(with_nills && err->part.range.with_nills) { + if(with_nills && err->with_nills) { msg = createException(SQL,"sql.alter_table_add_range_partition",SQLSTATE(42000) "ALTER TABLE: conflicting partitions: table %s.%s stores null values and only " "one partition can store null values at the time", err->t->s->base.name, err->t->base.name); @@ -343,15 +343,15 @@ finish: } static char * -alter_table_add_value_partition(mvc *sql, MalStkPtr stk, InstrPtr pci, char *msname, char *mtname, char *psname, char *ptname) +alter_table_add_value_partition(mvc *sql, MalStkPtr stk, InstrPtr pci, char *msname, char *mtname, char *psname, char *ptname, int with_nills) { sql_table *mt = NULL, *pt = NULL; str msg = MAL_SUCCEED, escaped = NULL; sql_column *col = NULL, *bcol = NULL; sql_part *err = NULL; int tp1 = 0, errcode = 0, i = 0, ninserts = 0; - BAT *b = NULL, *sorted = NULL, *cbind = NULL, *diff = NULL; - gdk_return ret = GDK_SUCCEED; + BAT *b = NULL, *cbind = NULL, *diff = NULL; + list *values = list_new(sql->sa, (fdestroy) NULL); int accesses[3] = {RDONLY, RD_INS, RD_UPD_VAL}; if((msg = validate_alter_table_add_table(sql, "sql.alter_table_add_value_partition", msname, mtname, psname, ptname, &mt, &pt))) @@ -365,8 +365,8 @@ alter_table_add_value_partition(mvc *sql col = mt->pcol; tp1 = col->type.type->localtype; - ninserts = pci->argc - pci->retc + 4; - if(ninserts <= 0) { + ninserts = pci->argc - pci->retc - 5; + if(ninserts <= 0 && !with_nills) { msg = createException(SQL,"sql.alter_table_add_value_partition",SQLSTATE(42000) "ALTER TABLE: no values in the list"); goto finish; } @@ -376,17 +376,24 @@ alter_table_add_value_partition(mvc *sql goto finish; } - for( i = pci->retc+4; i < pci->argc; i++){ + if (with_nills && BUNappend(b, ATOMnilptr(tp1), FALSE) != GDK_SUCCEED) { + msg = createException(SQL,"sql.alter_table_add_value_partition",SQLSTATE(HY001) MAL_MALLOC_FAIL); + goto finish; + } + for( i = pci->retc+5; i < pci->argc; i++){ ptr pnext = NULL; size_t len = 0; str next = *getArgReference_str(stk, pci, i); + sql_part_value *nextv = NULL; + void *prev; + if(escaped) { GDKfree(escaped); escaped = NULL; } - if(tp1 == TYPE_str && ATOMcmp(tp1, next, ATOMnilptr(tp1))) { - if ((escaped = add_quotes(next)) == NULL) { + if(tp1 == TYPE_str && ATOMcmp(tp1, next, ATOMnilptr(tp1)) == 0) { + if ((escaped = add_quotes(next)) == NULL) { /* escape string atoms properly */ msg = createException(SQL, "sql.alter_table_add_value_partition", SQLSTATE(HY001) MAL_MALLOC_FAIL); goto finish; } @@ -402,20 +409,34 @@ alter_table_add_value_partition(mvc *sql goto finish; } } - ret = BUNappend(b, pnext, FALSE); - GDKfree(pnext); - if (ret != GDK_SUCCEED) { + + if(ATOMcmp(tp1, pnext, ATOMnilptr(tp1)) == 0) { /* check for an eventual null value which cannot be */ + GDKfree(pnext); + msg = createException(SQL,"sql.alter_table_add_value_partition",SQLSTATE(42000) + "ALTER TABLE: list value cannot be null"); + goto finish; + } else if (BUNappend(b, pnext, FALSE) != GDK_SUCCEED) { + GDKfree(pnext); msg = createException(SQL,"sql.alter_table_add_value_partition",SQLSTATE(HY001) MAL_MALLOC_FAIL); goto finish; } + + nextv = SA_ZNEW(sql->session->tr->sa, sql_part_value); /* instantiate the part value */ + nextv->tpe = tp1; + nextv->value = sa_alloc(sql->session->tr->sa, len); + memcpy(nextv->value, pnext, len); + nextv->length = len; + + if((prev = list_append_sorted(values, nextv, sql_values_list_element_validate_and_insert)) != NULL) { + GDKfree(pnext); + msg = createException(SQL,"sql.alter_table_add_value_partition",SQLSTATE(42000) + "ALTER TABLE: there are duplicated values in the list"); + goto finish; + } + GDKfree(pnext); } - if (BATsort(&sorted, NULL, NULL, b, NULL, NULL, 0, 0) != GDK_SUCCEED) { - msg = createException(SQL,"sql.alter_table_add_value_partition",SQLSTATE(HY001) MAL_MALLOC_FAIL); - goto finish; - } - - bcol = mvc_bind_column(sql, pt, col->base.name); + bcol = mvc_bind_column(sql, pt, col->base.name); /* check if the values in the column are proper to the partition */ for(i = 0 ; i < 3 ; i++) { if(cbind) { BBPunfix(cbind->batCacheid); @@ -429,18 +450,19 @@ alter_table_add_value_partition(mvc *sql msg = createException(SQL,"sql.alter_table_add_value_partition",SQLSTATE(HY001) MAL_MALLOC_FAIL); goto finish; } - if((diff = BATdiff(cbind, sorted, NULL, NULL, 0, BUN_NONE)) == NULL) { + if((diff = BATdiff(cbind, b, NULL, NULL, 0, BUN_NONE)) == NULL) { msg = createException(SQL,"sql.alter_table_add_range_partition",SQLSTATE(HY001) MAL_MALLOC_FAIL); goto finish; } if(BATcount(diff) > 0) { msg = createException(SQL,"sql.alter_table_add_value_partition",SQLSTATE(42000) - "ALTER TABLE: there are values in the column %s not according to the partition values list", col->base.name); + "ALTER TABLE: there are values in the column %s not according to the partition values list", + col->base.name); goto finish; } } - errcode = sql_trans_add_value_partition(sql->session->tr, mt, pt, tp1, sorted, &err); + errcode = sql_trans_add_value_partition(sql->session->tr, mt, pt, tp1, values, with_nills, &err); switch(errcode) { case 0: break; @@ -465,10 +487,6 @@ finish: BBPunfix(cbind->batCacheid); if(diff) BBPunfix(diff->batCacheid); - if(sorted && msg) - BBPunfix(sorted->batCacheid); - else if(sorted) - BBPretain(sorted->batCacheid); return msg; } @@ -1580,9 +1598,10 @@ SQLalter_add_value_partition(Client cntx char *mtname = SaveArgReference(stk, pci, 2); char *psname = SaveArgReference(stk, pci, 3); char *ptname = SaveArgReference(stk, pci, 4); + int with_nills = *getArgReference_int(stk, pci, 5); initcontext(); - msg = alter_table_add_value_partition(sql, stk, pci, sname, mtname, psname, ptname); + msg = alter_table_add_value_partition(sql, stk, pci, sname, mtname, psname, ptname, with_nills); return msg; } diff --git a/sql/backends/monet5/sqlcatalog.mal b/sql/backends/monet5/sqlcatalog.mal --- a/sql/backends/monet5/sqlcatalog.mal +++ b/sql/backends/monet5/sqlcatalog.mal @@ -151,7 +151,11 @@ pattern alter_add_range_partition( sname address SQLalter_add_range_partition comment "Catalog operation alter_add_range_partition"; -pattern alter_add_value_partition( sname:str, mtnme:str, psnme:str, ptnme:str, arg:str...) +pattern alter_add_value_partition( sname:str, mtnme:str, psnme:str, ptnme:str, nills:int) +address SQLalter_add_value_partition +comment "Catalog operation alter_add_value_partition"; + +pattern alter_add_value_partition( sname:str, mtnme:str, psnme:str, ptnme:str, nills:int, arg:str...) address SQLalter_add_value_partition comment "Catalog operation alter_add_value_partition"; diff --git a/sql/backends/monet5/wlr.mal b/sql/backends/monet5/wlr.mal --- a/sql/backends/monet5/wlr.mal +++ b/sql/backends/monet5/wlr.mal @@ -251,6 +251,10 @@ unsafe pattern transaction() address WLRgeneric comment "Start an autocommit transaction"; -pattern alter_add_value_partition( sname:str, mtnme:str, psnme:str, ptnme:str, arg:str...) +pattern alter_add_value_partition( sname:str, mtnme:str, psnme:str, ptnme:str, nills:int) address WLRgeneric comment "Catalog operation alter_add_value_partition"; + +pattern alter_add_value_partition( sname:str, mtnme:str, psnme:str, ptnme:str, nills:int, arg:str...) +address WLRgeneric +comment "Catalog operation alter_add_value_partition"; diff --git a/sql/common/sql_list.c b/sql/common/sql_list.c --- a/sql/common/sql_list.c +++ b/sql/common/sql_list.c @@ -217,8 +217,7 @@ list_append_sorted(list *l, void *data, if(first) { n->next = l->h; l->h = n; - } - if(!m) { + } else if(!m) { l->t->next = n; l->t = n; } else diff --git a/sql/include/sql_catalog.h b/sql/include/sql_catalog.h --- a/sql/include/sql_catalog.h +++ b/sql/include/sql_catalog.h @@ -512,15 +512,21 @@ typedef enum table_types { #define TABLE_READONLY 1 #define TABLE_APPENDONLY 2 +typedef struct sql_part_value { + sht tpe; + ptr value; + size_t length; +} sql_part_value; + typedef struct sql_part { sql_base base; struct sql_table *t; /* cached value of the merge table */ sht tpe; /* the column type */ sht part_type; /* by range, list/values or none */ + int with_nills; union { - bat values; /* partition by values/list */ + list *values; /* partition by values/list */ struct sql_range { /* partition by range */ - int with_nills; ptr minvalue; ptr maxvalue; size_t minlength; @@ -634,6 +640,7 @@ extern sql_func *sql_trans_bind_func(sql extern sql_func *sql_trans_find_func(sql_trans *tr, int id); extern node *find_sql_func_node(sql_schema *s, int id); +extern void *sql_values_list_element_validate_and_insert(void *v1, void *v2, int* res); extern void *sql_range_part_validate_and_insert(void *v1, void *v2, int* res); extern void *sql_values_part_validate_and_insert(void *v1, void *v2); diff --git a/sql/server/rel_optimizer.c b/sql/server/rel_optimizer.c --- a/sql/server/rel_optimizer.c +++ b/sql/server/rel_optimizer.c @@ -8248,7 +8248,7 @@ rel_merge_table_rewrite(int *changes, mv e2 = create_table_part_atom_exp(sql, pt->tpe, pt->part.range.maxvalue); dup = rel_compare_exp_(sql, dup, le, e1, e2, 3, 0); - if(pt->part.range.with_nills) { /* handle the nulls case */ + if(pt->with_nills) { /* handle the nulls case */ sql_rel* extra; sql_exp *nils = rel_unop_(sql, le, NULL, "isnull", card_value); nils = exp_compare(sql->sa, nils, exp_atom_bool(sql->sa, 1), cmp_equal); @@ -8257,8 +8257,20 @@ rel_merge_table_rewrite(int *changes, mv dup = rel_or(sql, NULL, dup, extra, NULL, NULL, NULL); } } else if(isListPartitionTable(t)) { - list *exps = new_exp_list(sql->sa); /* TODO the list should come from the partition itself */ - sql_exp *ein = exp_in(sql->sa, le, exps, cmp_in); + sql_exp *ein; + list *exps = new_exp_list(sql->sa); + + for(node *n = pt->part.values->h ; n ; n = n->next) { + sql_part_value *next = (sql_part_value*) n->data; + sql_exp *e1 = create_table_part_atom_exp(sql, next->tpe, next->value); + list_append(exps, e1); + } + + if(pt->with_nills) { /* handle the nulls case */ + sql_exp *e2 = exp_atom(sql->sa, atom_general(sql->sa, &(t->pcol->type), NULL)); + list_append(exps, e2); + } + ein = exp_in(sql->sa, le, exps, cmp_in); dup = rel_select(sql->sa, dup, ein); } else { assert(0); diff --git a/sql/server/rel_schema.c b/sql/server/rel_schema.c --- a/sql/server/rel_schema.c +++ b/sql/server/rel_schema.c @@ -115,11 +115,11 @@ rel_alter_table_add_partition_range(sql_ } static sql_rel * -rel_alter_table_add_partition_list(sql_allocator *sa, char *sname, char *tname, char *sname2, char *tname2, dlist* ll) +rel_alter_table_add_partition_list(sql_allocator *sa, char *sname, char *tname, char *sname2, char *tname2, + int with_nils, list* ll) { _______________________________________________ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list