Changeset: 7f2765253f35 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/7f2765253f35
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
gdk/gdk_aggr.c
gdk/gdk_calc.h
monetdb5/modules/kernel/algebra.c
Branch: pushcands
Log Message:
aggr.{stdev,variance}{,p} now have optional candidate lists.
diffs (truncated from 365 to 300 lines):
diff --git a/clients/Tests/MAL-signatures.stable.out
b/clients/Tests/MAL-signatures.stable.out
--- a/clients/Tests/MAL-signatures.stable.out
+++ b/clients/Tests/MAL-signatures.stable.out
@@ -242,6 +242,7 @@ stdout of test 'MAL-signatures` in direc
[ "aggr", "stdev", "command aggr.stdev(X_0:bat[:int],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRstdev3_dbl;", "" ]
[ "aggr", "stdev", "command aggr.stdev(X_0:bat[:lng],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRstdev3_dbl;", "" ]
[ "aggr", "stdev", "command aggr.stdev(X_0:bat[:sht],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRstdev3_dbl;", "" ]
+[ "aggr", "stdev", "command aggr.stdev(X_0:bat[:any_2],
X_1:bat[:oid]):dbl ", "ALGstdev2;", "" ]
[ "aggr", "stdev", "command aggr.stdev(X_0:bat[:any_2]):dbl ",
"ALGstdev;", "" ]
[ "aggr", "stdevp", "command aggr.stdevp(X_0:bat[:bte],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRstdevp3_dbl;", ""
]
[ "aggr", "stdevp", "command aggr.stdevp(X_0:bat[:dbl],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRstdevp3_dbl;", ""
]
@@ -249,6 +250,7 @@ stdout of test 'MAL-signatures` in direc
[ "aggr", "stdevp", "command aggr.stdevp(X_0:bat[:int],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRstdevp3_dbl;", ""
]
[ "aggr", "stdevp", "command aggr.stdevp(X_0:bat[:lng],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRstdevp3_dbl;", ""
]
[ "aggr", "stdevp", "command aggr.stdevp(X_0:bat[:sht],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRstdevp3_dbl;", ""
]
+[ "aggr", "stdevp", "command aggr.stdevp(X_0:bat[:any_2],
X_1:bat[:oid]):dbl ", "ALGstdevp2;", "" ]
[ "aggr", "stdevp", "command aggr.stdevp(X_0:bat[:any_2]):dbl ",
"ALGstdevp;", "" ]
[ "aggr", "str_group_concat", "command
aggr.str_group_concat(X_0:bat[:str], X_1:bat[:oid], X_2:bat[:any_1]):bat[:str]
", "AGGRstr_group_concat;", "" ]
[ "aggr", "str_group_concat", "command
aggr.str_group_concat(X_0:bat[:str], X_1:bat[:str], X_2:bat[:oid],
X_3:bat[:any_1]):bat[:str] ", "AGGRstr_group_concat_sep;", "" ]
@@ -606,6 +608,7 @@ stdout of test 'MAL-signatures` in direc
[ "aggr", "variance", "command aggr.variance(X_0:bat[:int],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRvariance3_dbl;", ""
]
[ "aggr", "variance", "command aggr.variance(X_0:bat[:lng],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRvariance3_dbl;", ""
]
[ "aggr", "variance", "command aggr.variance(X_0:bat[:sht],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRvariance3_dbl;", ""
]
+[ "aggr", "variance", "command aggr.variance(X_0:bat[:any_2],
X_1:bat[:oid]):dbl ", "ALGvariance2;", "" ]
[ "aggr", "variance", "command aggr.variance(X_0:bat[:any_2]):dbl ",
"ALGvariance;", "" ]
[ "aggr", "variancep", "command aggr.variancep(X_0:bat[:bte],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRvariancep3_dbl;", ""
]
[ "aggr", "variancep", "command aggr.variancep(X_0:bat[:dbl],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRvariancep3_dbl;", ""
]
@@ -613,6 +616,7 @@ stdout of test 'MAL-signatures` in direc
[ "aggr", "variancep", "command aggr.variancep(X_0:bat[:int],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRvariancep3_dbl;", ""
]
[ "aggr", "variancep", "command aggr.variancep(X_0:bat[:lng],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRvariancep3_dbl;", ""
]
[ "aggr", "variancep", "command aggr.variancep(X_0:bat[:sht],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRvariancep3_dbl;", ""
]
+[ "aggr", "variancep", "command aggr.variancep(X_0:bat[:any_2],
X_1:bat[:oid]):dbl ", "ALGvariancep2;", "" ]
[ "aggr", "variancep", "command aggr.variancep(X_0:bat[:any_2]):dbl ",
"ALGvariancep;", "" ]
[ "aggr", "zero_or_one", "command
aggr.zero_or_one(X_0:bat[:any_1]):any_1 ", "zero_or_one;", "" ]
[ "aggr", "zero_or_one", "command aggr.zero_or_one(X_0:bat[:any_1],
X_1:bit):any_1 ", "zero_or_one_error;", "" ]
diff --git a/clients/Tests/MAL-signatures.stable.out.int128
b/clients/Tests/MAL-signatures.stable.out.int128
--- a/clients/Tests/MAL-signatures.stable.out.int128
+++ b/clients/Tests/MAL-signatures.stable.out.int128
@@ -283,6 +283,7 @@ stdout of test 'MAL-signatures` in direc
[ "aggr", "stdev", "command aggr.stdev(X_0:bat[:int],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRstdev3_dbl;", "" ]
[ "aggr", "stdev", "command aggr.stdev(X_0:bat[:lng],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRstdev3_dbl;", "" ]
[ "aggr", "stdev", "command aggr.stdev(X_0:bat[:sht],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRstdev3_dbl;", "" ]
+[ "aggr", "stdev", "command aggr.stdev(X_0:bat[:any_2],
X_1:bat[:oid]):dbl ", "ALGstdev2;", "" ]
[ "aggr", "stdev", "command aggr.stdev(X_0:bat[:any_2]):dbl ",
"ALGstdev;", "" ]
[ "aggr", "stdevp", "command aggr.stdevp(X_0:bat[:bte],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRstdevp3_dbl;", ""
]
[ "aggr", "stdevp", "command aggr.stdevp(X_0:bat[:dbl],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRstdevp3_dbl;", ""
]
@@ -291,6 +292,7 @@ stdout of test 'MAL-signatures` in direc
[ "aggr", "stdevp", "command aggr.stdevp(X_0:bat[:int],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRstdevp3_dbl;", ""
]
[ "aggr", "stdevp", "command aggr.stdevp(X_0:bat[:lng],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRstdevp3_dbl;", ""
]
[ "aggr", "stdevp", "command aggr.stdevp(X_0:bat[:sht],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRstdevp3_dbl;", ""
]
+[ "aggr", "stdevp", "command aggr.stdevp(X_0:bat[:any_2],
X_1:bat[:oid]):dbl ", "ALGstdevp2;", "" ]
[ "aggr", "stdevp", "command aggr.stdevp(X_0:bat[:any_2]):dbl ",
"ALGstdevp;", "" ]
[ "aggr", "str_group_concat", "command
aggr.str_group_concat(X_0:bat[:str], X_1:bat[:oid], X_2:bat[:any_1]):bat[:str]
", "AGGRstr_group_concat;", "" ]
[ "aggr", "str_group_concat", "command
aggr.str_group_concat(X_0:bat[:str], X_1:bat[:str], X_2:bat[:oid],
X_3:bat[:any_1]):bat[:str] ", "AGGRstr_group_concat_sep;", "" ]
@@ -722,6 +724,7 @@ stdout of test 'MAL-signatures` in direc
[ "aggr", "variance", "command aggr.variance(X_0:bat[:int],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRvariance3_dbl;", ""
]
[ "aggr", "variance", "command aggr.variance(X_0:bat[:lng],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRvariance3_dbl;", ""
]
[ "aggr", "variance", "command aggr.variance(X_0:bat[:sht],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRvariance3_dbl;", ""
]
+[ "aggr", "variance", "command aggr.variance(X_0:bat[:any_2],
X_1:bat[:oid]):dbl ", "ALGvariance2;", "" ]
[ "aggr", "variance", "command aggr.variance(X_0:bat[:any_2]):dbl ",
"ALGvariance;", "" ]
[ "aggr", "variancep", "command aggr.variancep(X_0:bat[:bte],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRvariancep3_dbl;", ""
]
[ "aggr", "variancep", "command aggr.variancep(X_0:bat[:dbl],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRvariancep3_dbl;", ""
]
@@ -730,6 +733,7 @@ stdout of test 'MAL-signatures` in direc
[ "aggr", "variancep", "command aggr.variancep(X_0:bat[:int],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRvariancep3_dbl;", ""
]
[ "aggr", "variancep", "command aggr.variancep(X_0:bat[:lng],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRvariancep3_dbl;", ""
]
[ "aggr", "variancep", "command aggr.variancep(X_0:bat[:sht],
X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl] ", "AGGRvariancep3_dbl;", ""
]
+[ "aggr", "variancep", "command aggr.variancep(X_0:bat[:any_2],
X_1:bat[:oid]):dbl ", "ALGvariancep2;", "" ]
[ "aggr", "variancep", "command aggr.variancep(X_0:bat[:any_2]):dbl ",
"ALGvariancep;", "" ]
[ "aggr", "zero_or_one", "command
aggr.zero_or_one(X_0:bat[:any_1]):any_1 ", "zero_or_one;", "" ]
[ "aggr", "zero_or_one", "command aggr.zero_or_one(X_0:bat[:any_1],
X_1:bit):any_1 ", "zero_or_one_error;", "" ]
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -99,12 +99,12 @@ BAT *BATcalcorcst(BAT *b, const ValRecor
BAT *BATcalcrsh(BAT *b1, BAT *b2, BAT *s1, BAT *s2, bool abort_on_error);
BAT *BATcalcrshcst(BAT *b, const ValRecord *v, BAT *s, bool abort_on_error);
BAT *BATcalcsign(BAT *b, BAT *s);
-dbl BATcalcstdev_population(dbl *avgp, BAT *b);
-dbl BATcalcstdev_sample(dbl *avgp, BAT *b);
+dbl BATcalcstdev_population(dbl *avgp, BAT *b, BAT *s);
+dbl BATcalcstdev_sample(dbl *avgp, BAT *b, BAT *s);
BAT *BATcalcsub(BAT *b1, BAT *b2, BAT *s1, BAT *s2, int tp, bool
abort_on_error);
BAT *BATcalcsubcst(BAT *b, const ValRecord *v, BAT *s, int tp, bool
abort_on_error);
-dbl BATcalcvariance_population(dbl *avgp, BAT *b);
-dbl BATcalcvariance_sample(dbl *avgp, BAT *b);
+dbl BATcalcvariance_population(dbl *avgp, BAT *b, BAT *s);
+dbl BATcalcvariance_sample(dbl *avgp, BAT *b, BAT *s);
BAT *BATcalcxor(BAT *b1, BAT *b2, BAT *s1, BAT *s2);
BAT *BATcalcxorcst(BAT *b, const ValRecord *v, BAT *s);
bool BATcheckorderidx(BAT *b);
diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c
--- a/gdk/gdk_aggr.c
+++ b/gdk/gdk_aggr.c
@@ -4098,24 +4098,25 @@ BATgroupquantile_avg(BAT *b, BAT *g, BAT
/* ---------------------------------------------------------------------- */
/* standard deviation (both biased and non-biased) */
-#define AGGR_STDEV_SINGLE(TYPE) \
- do { \
- TYPE x; \
- for (i = 0; i < cnt; i++) { \
- x = ((const TYPE *) values)[i]; \
- if (is_##TYPE##_nil(x)) \
- continue; \
- n++; \
- delta = (dbl) x - mean; \
- mean += delta / n; \
- m2 += delta * ((dbl) x - mean); \
- if (isinf(m2)) \
- goto overflow; \
- } \
+#define AGGR_STDEV_SINGLE(TYPE) \
+ do { \
+ TYPE x; \
+ for (i = 0; i < ci->ncand; i++) { \
+ oid o = canditer_next(ci) - ci->hseq; \
+ x = ((const TYPE *) values)[o]; \
+ if (is_##TYPE##_nil(x)) \
+ continue; \
+ n++; \
+ delta = (dbl) x - mean; \
+ mean += delta / n; \
+ m2 += delta * ((dbl) x - mean); \
+ if (isinf(m2)) \
+ goto overflow; \
+ } \
} while (0)
static dbl
-calcvariance(dbl *restrict avgp, const void *restrict values, BUN cnt, int tp,
bool issample)
+calcvariance(dbl *restrict avgp, const void *restrict values, struct canditer
*ci, int tp, bool issample)
{
BUN n = 0, i;
dbl mean = 0;
@@ -4164,52 +4165,60 @@ calcvariance(dbl *restrict avgp, const v
}
dbl
-BATcalcstdev_population(dbl *avgp, BAT *b)
+BATcalcstdev_population(dbl *avgp, BAT *b, BAT *s)
{
lng t0 = 0;
+ struct canditer ci;
TRC_DEBUG_IF(ALGO) t0 = GDKusec();
+ canditer_init(&ci, b, s);
dbl v = calcvariance(avgp, (const void *) Tloc(b, 0),
- BATcount(b), b->ttype, false);
+ &ci, b->ttype, false);
TRC_DEBUG(ALGO, "b=" ALGOBATFMT " (" LLFMT " usec)\n",
ALGOBATPAR(b), GDKusec() - t0);
return is_dbl_nil(v) ? dbl_nil : sqrt(v);
}
dbl
-BATcalcstdev_sample(dbl *avgp, BAT *b)
+BATcalcstdev_sample(dbl *avgp, BAT *b, BAT *s)
{
lng t0 = 0;
+ struct canditer ci;
TRC_DEBUG_IF(ALGO) t0 = GDKusec();
+ canditer_init(&ci, b, s);
dbl v = calcvariance(avgp, (const void *) Tloc(b, 0),
- BATcount(b), b->ttype, true);
+ &ci, b->ttype, true);
TRC_DEBUG(ALGO, "b=" ALGOBATFMT " (" LLFMT " usec)\n",
ALGOBATPAR(b), GDKusec() - t0);
return is_dbl_nil(v) ? dbl_nil : sqrt(v);
}
dbl
-BATcalcvariance_population(dbl *avgp, BAT *b)
+BATcalcvariance_population(dbl *avgp, BAT *b, BAT *s)
{
lng t0 = 0;
+ struct canditer ci;
TRC_DEBUG_IF(ALGO) t0 = GDKusec();
+ canditer_init(&ci, b, s);
dbl v = calcvariance(avgp, (const void *) Tloc(b, 0),
- BATcount(b), b->ttype, false);
+ &ci, b->ttype, false);
TRC_DEBUG(ALGO, "b=" ALGOBATFMT " (" LLFMT " usec)\n",
ALGOBATPAR(b), GDKusec() - t0);
return v;
}
dbl
-BATcalcvariance_sample(dbl *avgp, BAT *b)
+BATcalcvariance_sample(dbl *avgp, BAT *b, BAT *s)
{
lng t0 = 0;
+ struct canditer ci;
TRC_DEBUG_IF(ALGO) t0 = GDKusec();
+ canditer_init(&ci, b, s);
dbl v = calcvariance(avgp, (const void *) Tloc(b, 0),
- BATcount(b), b->ttype, true);
+ &ci, b->ttype, true);
TRC_DEBUG(ALGO, "b=" ALGOBATFMT " (" LLFMT " usec)\n",
ALGOBATPAR(b), GDKusec() - t0);
return v;
diff --git a/gdk/gdk_calc.h b/gdk/gdk_calc.h
--- a/gdk/gdk_calc.h
+++ b/gdk/gdk_calc.h
@@ -148,12 +148,12 @@ gdk_export void *BATmin(BAT *b, void *ag
gdk_export void *BATmax_skipnil(BAT *b, void *aggr, bit skipnil);
gdk_export void *BATmin_skipnil(BAT *b, void *aggr, bit skipnil);
-gdk_export dbl BATcalcstdev_population(dbl *avgp, BAT *b);
-gdk_export dbl BATcalcstdev_sample(dbl *avgp, BAT *b);
+gdk_export dbl BATcalcstdev_population(dbl *avgp, BAT *b, BAT *s);
+gdk_export dbl BATcalcstdev_sample(dbl *avgp, BAT *b, BAT *s);
gdk_export BAT *BATgroupstdev_sample(BAT *b, BAT *g, BAT *e, BAT *s, int tp,
bool skip_nils, bool abort_on_error);
gdk_export BAT *BATgroupstdev_population(BAT *b, BAT *g, BAT *e, BAT *s, int
tp, bool skip_nils, bool abort_on_error);
-gdk_export dbl BATcalcvariance_population(dbl *avgp, BAT *b);
-gdk_export dbl BATcalcvariance_sample(dbl *avgp, BAT *b);
+gdk_export dbl BATcalcvariance_population(dbl *avgp, BAT *b, BAT *s);
+gdk_export dbl BATcalcvariance_sample(dbl *avgp, BAT *b, BAT *s);
gdk_export BAT *BATgroupvariance_sample(BAT *b, BAT *g, BAT *e, BAT *s, int
tp, bool skip_nils, bool abort_on_error);
gdk_export BAT *BATgroupvariance_population(BAT *b, BAT *g, BAT *e, BAT *s,
int tp, bool skip_nils, bool abort_on_error);
gdk_export dbl BATcalccovariance_sample(BAT *b1, BAT *b2, BAT *s1, BAT *s2);
diff --git a/monetdb5/modules/kernel/algebra.c
b/monetdb5/modules/kernel/algebra.c
--- a/monetdb5/modules/kernel/algebra.c
+++ b/monetdb5/modules/kernel/algebra.c
@@ -1295,73 +1295,79 @@ ALGreuse(bat *ret, const bat *bid)
}
/*
- * BAT standard deviation
+ * BAT standard deviation and variance
*/
static str
+ALGstdvar(dbl *res, const bat *bid, const bat *sid,
+ dbl (*func)(dbl *, BAT *, BAT *), const char *malfunc)
+{
+ BAT *b, *s = NULL;
+ dbl result;
+
+ if ((b = BATdescriptor(*bid)) == NULL ||
+ (sid && !is_bat_nil(*sid) && (s = BATdescriptor(*sid)) ==
NULL)) {
+ if (b)
+ BBPunfix(b->batCacheid);
+ throw(MAL, malfunc, SQLSTATE(HY002) RUNTIME_OBJECT_MISSING);
+ }
+ result = (*func)(NULL, b, s);
+ BBPunfix(b->batCacheid);
+ if (s)
+ BBPunfix(s->batCacheid);
+ if (is_dbl_nil(result) && GDKerrbuf && GDKerrbuf[0])
+ throw(MAL, malfunc, GDK_EXCEPTION);
+ *res = result;
+ return MAL_SUCCEED;
+}
+
+static str
ALGstdev(dbl *res, const bat *bid)
{
- BAT *b;
- dbl stdev;
+ return ALGstdvar(res, bid, NULL, BATcalcstdev_sample, "aggr.stdev");
+}
- if ((b = BATdescriptor(*bid)) == NULL)
- throw(MAL, "aggr.stdev", SQLSTATE(HY002)
RUNTIME_OBJECT_MISSING);
- stdev = BATcalcstdev_sample(NULL, b);
- BBPunfix(b->batCacheid);
- if (is_dbl_nil(stdev) && GDKerrbuf && GDKerrbuf[0])
- throw(MAL, "aggr.stdev", GDK_EXCEPTION);
- *res = stdev;
- return MAL_SUCCEED;
+static str
+ALGstdev2(dbl *res, const bat *bid, const bat *sid)
+{
+ return ALGstdvar(res, bid, sid, BATcalcstdev_sample, "aggr.stdev");
}
static str
ALGstdevp(dbl *res, const bat *bid)
{
- BAT *b;
- dbl stdev;
-
- if ((b = BATdescriptor(*bid)) == NULL)
- throw(MAL, "aggr.stdevp", SQLSTATE(HY002)
RUNTIME_OBJECT_MISSING);
- stdev = BATcalcstdev_population(NULL, b);
- BBPunfix(b->batCacheid);
- if (is_dbl_nil(stdev) && GDKerrbuf && GDKerrbuf[0])
- throw(MAL, "aggr.stdevp", GDK_EXCEPTION);
- *res = stdev;
- return MAL_SUCCEED;
+ return ALGstdvar(res, bid, NULL, BATcalcstdev_population,
"aggr.stdevp");
}
-/*
- * BAT variance
- */
+static str
+ALGstdevp2(dbl *res, const bat *bid, const bat *sid)
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
