Changeset: ec0c2fd5efca for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ec0c2fd5efca
Added Files:
        sql/test/merge-partitions/Tests/mergepart02.stable.err
        sql/test/merge-partitions/Tests/mergepart02.stable.out
Modified Files:
        sql/backends/monet5/sql_cat.c
        sql/common/sql_changeset.c
        sql/common/sql_list.c
        sql/include/sql_catalog.h
        sql/include/sql_list.h
        sql/storage/sql_catalog.c
        sql/storage/sql_storage.h
        sql/storage/store.c
        sql/test/merge-partitions/Tests/All
        sql/test/merge-partitions/Tests/mergepart02.sql
Branch: merge-partitions
Log Message:

Calling BATsemijoin to validate partitions by list of values


diffs (truncated from 508 to 300 lines):

diff --git a/sql/backends/monet5/sql_cat.c b/sql/backends/monet5/sql_cat.c
--- a/sql/backends/monet5/sql_cat.c
+++ b/sql/backends/monet5/sql_cat.c
@@ -184,10 +184,12 @@ alter_table_add_range_partition(mvc *sql
                        msg = 
createException(SQL,"sql.alter_table_add_range_partition",SQLSTATE(HY001) 
MAL_MALLOC_FAIL);
                        break;
                case -2:
-                       msg = 
createException(SQL,"sql.alter_table_add_range_partition",SQLSTATE(42000) 
"ALTER TABLE: minimum value length is higher than %d", 
STORAGE_MAX_VALUE_LENGTH);
+                       msg = 
createException(SQL,"sql.alter_table_add_range_partition",SQLSTATE(42000)
+                                                                       "ALTER 
TABLE: minimum value length is higher than %d", STORAGE_MAX_VALUE_LENGTH);
                        break;
                case -3:
-                       msg = 
createException(SQL,"sql.alter_table_add_range_partition",SQLSTATE(42000) 
"ALTER TABLE: maximum value length is higher than %d", 
STORAGE_MAX_VALUE_LENGTH);
+                       msg = 
createException(SQL,"sql.alter_table_add_range_partition",SQLSTATE(42000)
+                                                                       "ALTER 
TABLE: maximum value length is higher than %d", STORAGE_MAX_VALUE_LENGTH);
                        break;
                case -4:
                        assert(err);
@@ -198,7 +200,8 @@ alter_table_add_range_partition(mvc *sql
                                GDKfree(err_min);
                                msg = 
createException(SQL,"sql.alter_table_add_range_partition",SQLSTATE(HY001) 
MAL_MALLOC_FAIL);
                        } else {
-                               msg = 
createException(SQL,"sql.alter_table_add_range_partition",SQLSTATE(42000) 
"ALTER TABLE: conflicting partitions: %s to %s and %s to %s from table %s.%s",
+                               msg = 
createException(SQL,"sql.alter_table_add_range_partition",SQLSTATE(42000)
+                                                                 "ALTER TABLE: 
conflicting partitions: %s to %s and %s to %s from table %s.%s",
                                                                  min, max, 
err_min, err_max, err->t->s->base.name, err->t->base.name);
                                GDKfree(err_min);
                                GDKfree(err_max);
@@ -222,6 +225,7 @@ alter_table_add_value_partition(mvc *sql
        sql_table *mt = NULL, *pt = NULL;
        str msg = MAL_SUCCEED;
        sql_column *col = NULL;
+       sql_part *err = NULL;
        int tp1 = 0, errcode = 0, i = 0, ninserts = 0;
        BAT *b = NULL;
        gdk_return ret = GDK_SUCCEED;
@@ -254,7 +258,8 @@ alter_table_add_value_partition(mvc *sql
                str next = *getArgReference_str(stk, pci, i);
 
                if(ATOMfromstr(tp1, &pnext, &len, next) < 0) {
-                       msg = 
createException(SQL,"sql.alter_table_add_value_partition",SQLSTATE(42000) 
"ALTER TABLE: error while parsing value %s", next);
+                       msg = 
createException(SQL,"sql.alter_table_add_value_partition",SQLSTATE(42000)
+                                                                       "ALTER 
TABLE: error while parsing value %s", next);
                        goto finish;
                }
                ret = BUNappend(b, pnext, FALSE);
@@ -265,16 +270,19 @@ alter_table_add_value_partition(mvc *sql
                }
        }
 
-       /* TODO search for a conflicting partition */
-       errcode = sql_trans_add_value_partition(sql->session->tr, mt, pt, tp1, 
b);
+       errcode = sql_trans_add_value_partition(sql->session->tr, mt, pt, tp1, 
b, &err);
        switch(errcode) {
                case 0:
                        break;
                case -1:
-                       msg = 
createException(SQL,"sql.alter_table_add_value_partition",SQLSTATE(HY001) 
MAL_MALLOC_FAIL);
+                       msg = 
createException(SQL,"sql.alter_table_add_value_partition",SQLSTATE(42000)
+                                                                       "ALTER 
TABLE: the new partition is conflicting with the existing partition %s.%s",
+                                                                       
err->t->s->base.name, err->t->base.name);
                        break;
                default:
-                       msg = 
createException(SQL,"sql.alter_table_add_value_partition",SQLSTATE(42000) 
"ALTER TABLE: value at position %d length is higher than %d", (errcode * -1) - 
1, STORAGE_MAX_VALUE_LENGTH);
+                       msg = 
createException(SQL,"sql.alter_table_add_value_partition",SQLSTATE(42000) \
+                                                                       "ALTER 
TABLE: value at position %d length is higher than %d",
+                                                                       
(errcode * -1) - 1, STORAGE_MAX_VALUE_LENGTH);
                        break;
        }
 
diff --git a/sql/common/sql_changeset.c b/sql/common/sql_changeset.c
--- a/sql/common/sql_changeset.c
+++ b/sql/common/sql_changeset.c
@@ -43,6 +43,19 @@ cs_add(changeset * cs, void *elm, int fl
 }
 
 void*
+cs_add_with_validate(changeset * cs, void *elm, int flag, fvalidate cmp)
+{
+       void* res = NULL;
+       if (!cs->set)
+               cs->set = list_new(cs->sa, cs->destroy);
+       if((res = list_append_with_validate(cs->set, elm, cmp)) != NULL)
+               return res;
+       if (flag == TR_NEW && !cs->nelm)
+               cs->nelm = cs->set->t;
+       return res;
+}
+
+void*
 cs_add_sorted(changeset * cs, void *elm, int flag, fcmpvalidate cmp)
 {
        void* res = NULL;
diff --git a/sql/common/sql_list.c b/sql/common/sql_list.c
--- a/sql/common/sql_list.c
+++ b/sql/common/sql_list.c
@@ -161,6 +161,39 @@ list_append(list *l, void *data)
 }
 
 void*
+list_append_with_validate(list *l, void *data, fvalidate cmp)
+{
+       node *n = node_create(l->sa, data), *m;
+       void* err = NULL;
+
+       if (n == NULL)
+               return NULL;
+       if (l->cnt) {
+               for (m = l->h; m; m = m->next) {
+                       err = cmp(m->data, data);
+                       if(err)
+                               return err;
+               }
+               l->t->next = n;
+       } else {
+               l->h = n;
+       }
+       l->t = n;
+       l->cnt++;
+       MT_lock_set(&l->ht_lock);
+       if (l->ht) {
+               int key = l->ht->key(data);
+
+               if (hash_add(l->ht, key, data) == NULL) {
+                       MT_lock_unset(&l->ht_lock);
+                       return NULL;
+               }
+       }
+       MT_lock_unset(&l->ht_lock);
+       return NULL;
+}
+
+void*
 list_append_sorted(list *l, void *data, fcmpvalidate cmp)
 {
        node *n = node_create(l->sa, data), *m;
diff --git a/sql/include/sql_catalog.h b/sql/include/sql_catalog.h
--- a/sql/include/sql_catalog.h
+++ b/sql/include/sql_catalog.h
@@ -201,6 +201,7 @@ typedef struct changeset {
 extern void cs_new(changeset * cs, sql_allocator *sa, fdestroy destroy);
 extern void cs_destroy(changeset * cs);
 extern void cs_add(changeset * cs, void *elm, int flag);
+extern void *cs_add_with_validate(changeset * cs, void *elm, int flag, 
fvalidate cmp);
 extern void *cs_add_sorted(changeset * cs, void *elm, int flag, fcmpvalidate 
cmp);
 extern void cs_add_before(changeset * cs, node *n, void *elm);
 extern void cs_del(changeset * cs, node *elm, int flag);
@@ -630,6 +631,7 @@ extern sql_func *sql_trans_find_func(sql
 extern node *find_sql_func_node(sql_schema *s, int id);
 
 extern void *sql_range_part_validate_and_insert(void *v1, void *v2, int* res);
+extern void *sql_values_part_validate_and_insert(void *v1, void *v2);
 
 typedef struct {
        BAT *b;
diff --git a/sql/include/sql_list.h b/sql/include/sql_list.h
--- a/sql/include/sql_list.h
+++ b/sql/include/sql_list.h
@@ -59,12 +59,14 @@ extern int list_traverse(list *l, traver
  * */
 typedef int (*fcmp) (void *data, void *key);
 typedef void *(*fcmpvalidate) (void *v1, void *v2, int *cmp);
+typedef void *(*fvalidate) (void *v1, void *v2);
 typedef int (*fcmp2) (void *data, void *v1, void *v2);
 typedef void *(*fdup) (void *data);
 typedef void *(*freduce) (void *v1, void *v2);
 typedef void *(*freduce2) (sql_allocator *sa, void *v1, void *v2);
 typedef void *(*fmap) (void *data, void *clientdata);
 
+extern void *list_append_with_validate(list *l, void *data, fvalidate cmp);
 extern void *list_append_sorted(list *l, void *data, fcmpvalidate cmp);
 extern node *list_find(list *l, void *key, fcmp cmp);
 extern int  list_position(list *l, void *val);
diff --git a/sql/storage/sql_catalog.c b/sql/storage/sql_catalog.c
--- a/sql/storage/sql_catalog.c
+++ b/sql/storage/sql_catalog.c
@@ -358,3 +358,33 @@ sql_range_part_validate_and_insert(void 
        *res = res2;
        return NULL;
 }
+
+void*
+sql_values_part_validate_and_insert(void *v1, void *v2)
+{
+       sql_part* pt = (sql_part*) v1, *newp = (sql_part*) v2, *res = NULL;
+       BAT* b1 = NULL, *b2 = NULL, *b3  = NULL;
+
+       if ((b1 = BATdescriptor(pt->part.values)) == NULL) {
+               res = pt;
+               goto finish;
+       }
+       if ((b2 = BATdescriptor(newp->part.values)) == NULL) {
+               res = pt;
+               goto finish;
+       }
+       if(BATsemijoin(&b3, NULL, b1, b2, NULL, NULL, 0, BUN_NONE) != 
GDK_SUCCEED) {
+               res = pt;
+               goto finish;
+       }
+       if(BATcount(b3) > 0)
+               res = pt;
+finish:
+       if(b1)
+               BBPunfix(b1->batCacheid);
+       if(b2)
+               BBPunfix(b2->batCacheid);
+       if(b3)
+               BBPunfix(b3->batCacheid);
+       return res;
+}
diff --git a/sql/storage/sql_storage.h b/sql/storage/sql_storage.h
--- a/sql/storage/sql_storage.h
+++ b/sql/storage/sql_storage.h
@@ -361,7 +361,7 @@ extern sql_table *sql_trans_create_table
 extern int sql_trans_set_partition_table(sql_trans *tr, sql_table *t);
 extern sql_table *sql_trans_add_table(sql_trans *tr, sql_table *mt, sql_table 
*pt);
 extern int sql_trans_add_range_partition(sql_trans *tr, sql_table *mt, 
sql_table *pt, int tpe, ptr min, size_t smin, ptr max, size_t smax, sql_part** 
err);
-extern int sql_trans_add_value_partition(sql_trans *tr, sql_table *mt, 
sql_table *pt, int tpe, BAT* b);
+extern int sql_trans_add_value_partition(sql_trans *tr, sql_table *mt, 
sql_table *pt, int tpe, BAT* b, sql_part **err);
 extern sql_table *sql_trans_del_table(sql_trans *tr, sql_table *mt, sql_table 
*pt, int drop_action);
 
 extern int sql_trans_drop_table(sql_trans *tr, sql_schema *s, int id, int 
drop_action);
diff --git a/sql/storage/store.c b/sql/storage/store.c
--- a/sql/storage/store.c
+++ b/sql/storage/store.c
@@ -816,6 +816,9 @@ load_table(sql_trans *tr, sql_schema *s,
                        if(isRangePartitionTable(t)) {
                                sql_part *err = cs_add_sorted(&t->members, pt, 
TR_OLD, sql_range_part_validate_and_insert);
                                assert(!err);
+                       } else if(isListPartitionTable(t)) {
+                               sql_part *err = 
cs_add_with_validate(&t->members, pt, TR_OLD, 
sql_values_part_validate_and_insert);
+                               assert(!err);
                        } else {
                                cs_add(&t->members, pt, TR_OLD);
                        }
@@ -1475,6 +1478,9 @@ dup_sql_part(sql_allocator *sa, sql_tabl
        if(isRangePartitionTable(ot)) {
                sql_part *err = cs_add_sorted(&mt->members, pt, TR_NEW, 
sql_range_part_validate_and_insert);
                assert(!err);
+       } else if(isListPartitionTable(ot)) {
+               sql_part *err = cs_add_with_validate(&mt->members, pt, TR_NEW, 
sql_values_part_validate_and_insert);
+               assert(!err);
        } else {
                cs_add(&mt->members, pt, TR_NEW);
        }
@@ -2636,6 +2642,9 @@ table_dup(sql_trans *tr, int flag, sql_t
                        if(isRangePartitionTable(ot)) {
                                sql_part *err = cs_add_sorted(&t->members, 
dupped, tr_flag(&pt->base, flag), sql_range_part_validate_and_insert);
                                assert(!err);
+                       } else if(isListPartitionTable(ot)) {
+                               sql_part *err = 
cs_add_with_validate(&t->members, dupped, tr_flag(&pt->base, flag), 
sql_values_part_validate_and_insert);
+                               assert(!err);
                        } else {
                                cs_add(&t->members, dupped, tr_flag(&pt->base, 
flag));
                        }
@@ -4681,7 +4690,7 @@ finish:
 }
 
 int
-sql_trans_add_value_partition(sql_trans *tr, sql_table *mt, sql_table *pt, int 
tpe, BAT* b)
+sql_trans_add_value_partition(sql_trans *tr, sql_table *mt, sql_table *pt, int 
tpe, BAT* b, sql_part **err)
 {
        sql_schema *syss = find_sql_schema(tr, isGlobal(mt)?"sys":"tmp");
        sql_table *sysobj = find_sql_table(syss, "objects");
@@ -4726,11 +4735,15 @@ sql_trans_add_value_partition(sql_trans 
        _DELETE(v);
        BATsetcount(b, i);
        BATsettrivprop(b);
+       p->part.values = b->batCacheid;
+
        /* add list partition values */
-       p->part.values = b->batCacheid;
+       *err = cs_add_with_validate(&mt->members, p, TR_NEW, 
sql_values_part_validate_and_insert);
+       if(*err)
+               return -1;
+
        BBPretain(b->batCacheid);
 
-       cs_add(&mt->members, p, TR_NEW);
        /* add merge table dependency */
        sql_trans_create_dependency(tr, pt->base.id, mt->base.id, 
TABLE_DEPENDENCY);
        table_funcs.table_insert(tr, sysobj, &mt->base.id, p->base.name, 
&p->base.id);
diff --git a/sql/test/merge-partitions/Tests/All 
b/sql/test/merge-partitions/Tests/All
--- a/sql/test/merge-partitions/Tests/All
+++ b/sql/test/merge-partitions/Tests/All
@@ -1,3 +1,4 @@
 mergepart00
 mergepart01
 mergepart02
+mergepart03
diff --git a/sql/test/merge-partitions/Tests/mergepart02.sql 
b/sql/test/merge-partitions/Tests/mergepart02.sql
--- a/sql/test/merge-partitions/Tests/mergepart02.sql
+++ b/sql/test/merge-partitions/Tests/mergepart02.sql
@@ -4,18 +4,23 @@ SELECT COUNT(*) from value_partitions;
 
 CREATE TABLE subtable1 (a int, b varchar(32));
 ALTER TABLE listparts ADD TABLE subtable1 AS PARTITION IN ('1', '2', '3');
+SELECT COUNT(*) from range_partitions;
 SELECT COUNT(*) from value_partitions;
 
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list

Reply via email to