MonetDB: dict - unsigned vs signed gives interesting problems.

[Niels Nes]

Changeset: 8f6d3e2a2eac for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/8f6d3e2a2eac
Modified Files:
        sql/backends/monet5/dict.c
Branch: dict
Log Message:

unsigned vs signed gives interesting problems.
set min/max based on overflow.


diffs (45 lines):

diff --git a/sql/backends/monet5/dict.c b/sql/backends/monet5/dict.c
--- a/sql/backends/monet5/dict.c
+++ b/sql/backends/monet5/dict.c
@@ -132,15 +132,17 @@ DICTcompress(Client cntxt, MalBlkPtr mb,
                        throw(SQL, "dict.compress", SQLSTATE(HY013) 
"alter_storage failed");
                }
                BUN minpos = BUN_NONE, maxpos = BUN_NONE;
+               bte m = BATcount(u) - 1;
+               bte minval = m<0?m:0;
+               bte maxval = m<0?127:m;
                BATloop(o, p, q) {
-                       if (op[p] == 0) {
+                       if (op[p] == minval) {
                                minpos = p;
                                break;
                        }
                }
-               bte m = BATcount(u) - 1;
                BATloop(o, p, q) {
-                       if (op[p] == m) {
+                       if (op[p] == maxval) {
                                maxpos = p;
                                break;
                        }
@@ -169,15 +171,17 @@ DICTcompress(Client cntxt, MalBlkPtr mb,
                        throw(SQL, "dict.compress", SQLSTATE(HY013) 
"alter_storage failed");
                }
                BUN minpos = BUN_NONE, maxpos = BUN_NONE;
+               sht m = BATcount(u) - 1;
+               sht minval = m<0?m:0;
+               sht maxval = m<0?32767:m;
                BATloop(o, p, q) {
-                       if (op[p] == 0) {
+                       if (op[p] == minval) {
                                minpos = p;
                                break;
                        }
                }
-               sht m = BATcount(u) - 1;
                BATloop(o, p, q) {
-                       if (op[p] == m) {
+                       if (op[p] == maxval) {
                                maxpos = p;
                                break;
                        }
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list