Changeset: d99800f98e70 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/d99800f98e70
Modified Files:
clients/Tests/MAL-signatures-hge.test
clients/mapilib/connect_openssl.c
cmake/monetdb-findpackages.cmake
gdk/gdk.h
gdk/gdk_string.c
monetdb5/modules/kernel/aggr.c
sql/ChangeLog
sql/backends/monet5/sql_upgrades.c
sql/scripts/49_strings.sql
sql/test/emptydb-previous-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out
sql/test/emptydb-previous-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/emptydb-previous-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-previous-upgrade/Tests/upgrade.stable.out
sql/test/emptydb-previous-upgrade/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out
sql/test/emptydb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/emptydb-upgrade/Tests/upgrade.stable.out
sql/test/emptydb-upgrade/Tests/upgrade.stable.out.int128
sql/test/emptydb/Tests/check.stable.out.int128
sql/test/testdb-previous-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-previous-upgrade-chain/Tests/upgrade.stable.out
sql/test/testdb-previous-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/testdb-previous-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-previous-upgrade/Tests/upgrade.stable.out
sql/test/testdb-previous-upgrade/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-chain-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out
sql/test/testdb-upgrade-chain/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade-hge/Tests/upgrade.stable.out.int128
sql/test/testdb-upgrade/Tests/upgrade.stable.out
sql/test/testdb-upgrade/Tests/upgrade.stable.out.int128
Branch: default
Log Message:
Implemented digest aggregates. See sql/ChangeLog.
diffs (truncated from 1285 to 300 lines):
diff --git a/clients/Tests/MAL-signatures-hge.test
b/clients/Tests/MAL-signatures-hge.test
--- a/clients/Tests/MAL-signatures-hge.test
+++ b/clients/Tests/MAL-signatures-hge.test
@@ -1044,6 +1044,36 @@ command aggr.quantile_avg(X_0:bat[:any_1
AGGRquantile_avg_cst
Quantile aggregate
aggr
+ripemd160
+command aggr.ripemd160(X_0:bat[:str]):str
+AGGRripemd160
+Ungrouped RIPEMD160
+aggr
+sha1
+command aggr.sha1(X_0:bat[:str]):str
+AGGRsha1
+Ungrouped SHA1
+aggr
+sha224
+command aggr.sha224(X_0:bat[:str]):str
+AGGRsha224
+Ungrouped SHA224
+aggr
+sha256
+command aggr.sha256(X_0:bat[:str]):str
+AGGRsha256
+Ungrouped SHA256
+aggr
+sha384
+command aggr.sha384(X_0:bat[:str]):str
+AGGRsha384
+Ungrouped SHA384
+aggr
+sha512
+command aggr.sha512(X_0:bat[:str]):str
+AGGRsha512
+Ungrouped SHA512
+aggr
stdev
command aggr.stdev(X_0:bat[:bte], X_1:bat[:oid], X_2:bat[:any_1]):bat[:dbl]
AGGRstdev3_dbl
@@ -2034,6 +2064,36 @@ command aggr.subquantile_avg(X_0:bat[:an
AGGRsubquantilecand_avg
Grouped quantile aggregate with candidate list
aggr
+subripemd160
+command aggr.subripemd160(X_0:bat[:str], X_1:bat[:oid], X_2:bat[:any_1],
X_3:bit):bat[:str]
+AGGRripemd160grouped
+Grouped RIPEMD160
+aggr
+subsha1
+command aggr.subsha1(X_0:bat[:str], X_1:bat[:oid], X_2:bat[:any_1],
X_3:bit):bat[:str]
+AGGRsha1grouped
+Grouped SHA1
+aggr
+subsha224
+command aggr.subsha224(X_0:bat[:str], X_1:bat[:oid], X_2:bat[:any_1],
X_3:bit):bat[:str]
+AGGRsha224grouped
+Grouped SHA224
+aggr
+subsha256
+command aggr.subsha256(X_0:bat[:str], X_1:bat[:oid], X_2:bat[:any_1],
X_3:bit):bat[:str]
+AGGRsha256grouped
+Grouped SHA256
+aggr
+subsha384
+command aggr.subsha384(X_0:bat[:str], X_1:bat[:oid], X_2:bat[:any_1],
X_3:bit):bat[:str]
+AGGRsha384grouped
+Grouped SHA384
+aggr
+subsha512
+command aggr.subsha512(X_0:bat[:str], X_1:bat[:oid], X_2:bat[:any_1],
X_3:bit):bat[:str]
+AGGRsha512grouped
+Grouped SHA512
+aggr
substdev
command aggr.substdev(X_0:bat[:bte], X_1:bat[:oid], X_2:bat[:any_1],
X_3:bit):bat[:dbl]
AGGRsubstdev_dbl
diff --git a/clients/mapilib/connect_openssl.c
b/clients/mapilib/connect_openssl.c
--- a/clients/mapilib/connect_openssl.c
+++ b/clients/mapilib/connect_openssl.c
@@ -139,7 +139,7 @@ verify_server_certificate_hash(Mapi mid,
}
assert(buf);
- // Compute the has of the DER using the deprecated API so we stay
+ // Compute the hash of the DER using the deprecated API so we stay
// compatible with OpenSSL 1.1.1.
SHA256_CTX sha256;
if (1 != SHA256_Init(&sha256)) {
diff --git a/cmake/monetdb-findpackages.cmake b/cmake/monetdb-findpackages.cmake
--- a/cmake/monetdb-findpackages.cmake
+++ b/cmake/monetdb-findpackages.cmake
@@ -126,7 +126,7 @@ if(WITH_MALLOC)
endif()
if(WITH_OPENSSL)
- find_package(OpenSSL 1.1.1)
+ find_package(OpenSSL 3.0.0)
endif()
find_package(Sphinx)
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -1549,6 +1549,9 @@ gdk_export BAT *BATtolower(BAT *b, BAT *
gdk_export BAT *BATcasefold(BAT *b, BAT *s);
gdk_export gdk_return GDKasciify(char **restrict buf, size_t *restrict buflen,
const char *restrict s);
gdk_export BAT *BATasciify(BAT *b, BAT *s);
+#ifdef HAVE_OPENSSL
+gdk_return BATaggrdigest(allocator *ma, BAT **bnp, char **shap, const char
*digest, BAT *b, BAT *g, BAT *e, BAT *s, bool skip_nils);
+#endif
gdk_export BAT *BATsample(BAT *b, BUN n);
gdk_export BAT *BATsample_with_seed(BAT *b, BUN n, uint64_t seed);
diff --git a/gdk/gdk_string.c b/gdk/gdk_string.c
--- a/gdk/gdk_string.c
+++ b/gdk/gdk_string.c
@@ -9688,3 +9688,122 @@ BATasciify(BAT *b, BAT *s)
BBPreclaim(bn);
return NULL;
}
+
+#ifdef HAVE_OPENSSL
+
+#include <openssl/evp.h>
+
+gdk_return
+BATaggrdigest(allocator *ma, BAT **bnp, char **shap, const char *digest,
+ BAT *b, BAT *g, BAT *e, BAT *s, bool skip_nils)
+{
+ oid min, max;
+ BUN ngrp;
+ struct canditer ci;
+ const char *err;
+ const oid *gids = g ? (const oid *) Tloc(g, 0) : NULL;
+ oid gid = 0;
+ QryCtx *qry_ctx = MT_thread_get_qry_ctx();
+ BAT *bn = NULL;
+
+ /* exactly one of bnp and shap should be non-NULL */
+ assert(bnp == NULL || shap == NULL);
+ assert(bnp != NULL || shap != NULL);
+
+ if ((err = BATgroupaggrinit(b, g, e, s, &min, &max, &ngrp, &ci)) !=
NULL) {
+ GDKerror("%s\n", err);
+ return GDK_FAIL;
+ }
+
+ if (bnp) {
+ if ((bn = COLnew(min, TYPE_str, ngrp, TRANSIENT)) == NULL)
+ return GDK_FAIL;
+ *bnp = bn;
+ }
+
+ BATiter bi = bat_iterator(b);
+
+ allocator *ta = MT_thread_getallocator();
+ allocator_state ta_state = ma_open(ta);
+
+ const EVP_MD *md;
+ md = EVP_get_digestbyname(digest);
+ EVP_MD_CTX **mdctx = ma_zalloc(ta, ngrp * sizeof(*mdctx));
+ if (mdctx == NULL)
+ goto bailout;
+
+ TIMEOUT_LOOP_IDX_DECL(i, ci.ncand, qry_ctx) {
+ oid p = canditer_next(&ci) - b->hseqbase;
+ if (gids)
+ gid = gids[p];
+ else if (g)
+ gid = p;
+ const char *s = BUNtvar(bi, p);
+ if (strNil(s)) {
+ if (!skip_nils) {
+ EVP_MD_CTX_free(mdctx[gid]);
+ mdctx[gid] = (EVP_MD_CTX *) -1;
+ }
+ continue;
+ }
+ if (mdctx[gid] == NULL) {
+ mdctx[gid] = EVP_MD_CTX_new();
+ if (mdctx[gid] == NULL ||
!EVP_DigestInit_ex2(mdctx[gid], md, NULL)) {
+ goto bailout;
+ }
+ } else if (mdctx[gid] == (EVP_MD_CTX *) -1) {
+ continue;
+ }
+ /* calculate digest including terminating NUL byte */
+ if (!EVP_DigestUpdate(mdctx[gid], s, strlen(s) + 1)) {
+ goto bailout;
+ }
+ }
+ TIMEOUT_CHECK(qry_ctx,
+ GOTO_LABEL_TIMEOUT_HANDLER(bailout, qry_ctx));
+
+ unsigned char md_value[EVP_MAX_MD_SIZE];
+ unsigned int md_len;
+ char digestbuf[EVP_MAX_MD_SIZE * 2 + 1];
+
+ for (gid = 0; gid < ngrp; gid++) {
+ if (mdctx[gid] == NULL || mdctx[gid] == (EVP_MD_CTX *) -1) {
+ strcpy_len(digestbuf, str_nil, sizeof(digestbuf));
+ } else {
+ if (!EVP_DigestFinal_ex(mdctx[gid], md_value, &md_len))
{
+ goto bailout;
+ }
+ for (unsigned int x = 0; x < md_len; x++) {
+ digestbuf[x * 2] =
"0123456789abcdef"[md_value[x] >> 4];
+ digestbuf[x * 2 + 1] =
"0123456789abcdef"[md_value[x] & 0xF];
+ }
+ digestbuf[2 * md_len] = 0;
+ EVP_MD_CTX_free(mdctx[gid]);
+ }
+ if (bnp) {
+ if (BUNappend(bn, digestbuf, false) != GDK_SUCCEED) {
+ goto bailout;
+ }
+ } else {
+ if ((*shap = ma_strdup(ma, digestbuf)) == NULL) {
+ goto bailout;
+ }
+ }
+ }
+ bat_iterator_end(&bi);
+ ma_close(ta, &ta_state);
+ return GDK_SUCCEED;
+
+ bailout:
+ bat_iterator_end(&bi);
+ if (mdctx) {
+ for (gid = 0; gid < ngrp; gid++)
+ if (mdctx[gid] != (EVP_MD_CTX *) -1)
+ EVP_MD_CTX_free(mdctx[gid]);
+ }
+ ma_close(ta, &ta_state);
+ BBPreclaim(bn);
+ return GDK_FAIL;
+}
+
+#endif
diff --git a/monetdb5/modules/kernel/aggr.c b/monetdb5/modules/kernel/aggr.c
--- a/monetdb5/modules/kernel/aggr.c
+++ b/monetdb5/modules/kernel/aggr.c
@@ -1271,6 +1271,145 @@ AGGRsubstr_group_concat_sep(Client ctx,
}
static str
+grpdigest(Client ctx,
+ bat *retval1, str *retval2, /* one or the other! */
+ const bat *bid, const bat *gid,
+ const bat *eid, bool skip_nils,
+ const char *digest,
+ const char *malfunc)
+{
+#ifdef HAVE_OPENSSL
+ BAT *b = BATdescriptor(*bid);
+ BAT *g = gid ? BATdescriptor(*gid) : NULL;
+ BAT *e = eid ? BATdescriptor(*eid) : NULL;
+ BAT *bn = NULL;
+
+ /* one or the other, not both */
+ assert(retval1 == NULL || retval2 == NULL);
+ assert(retval1 != NULL || retval2 != NULL);
+
+ if (b == NULL || (gid != NULL && g == NULL) || (eid != NULL && e ==
NULL)) {
+ BBPreclaim(b);
+ BBPreclaim(g);
+ BBPreclaim(e);
+ throw(MAL, malfunc, SQLSTATE(HY002) RUNTIME_OBJECT_MISSING);
+ }
+ if (BATaggrdigest(ctx->curprg->def->ma,
+ retval1 ? &bn : NULL, retval2,
+ digest,
+ b, g, e, NULL, skip_nils) !=
GDK_SUCCEED) {
+ BBPreclaim(b);
+ BBPreclaim(g);
+ BBPreclaim(e);
+ throw(MAL, malfunc, GDK_EXCEPTION);
+ }
+ if (retval1) {
+ *retval1 = bn->batCacheid;
+ BBPkeepref(bn);
+ }
+ BBPreclaim(b);
+ BBPreclaim(g);
+ BBPreclaim(e);
+ return MAL_SUCCEED;
+#else
+ (void) ctx;
+ (void) retval1;
+ (void) retval2;
+ (void) bid;
+ (void) gid;
+ (void) eid;
+ (void) skip_nils;
+ (void) digest;
+ throw(MAL, malfunc,
+ SQLSTATE(0A000) PROGRAM_NYI ": no OpenSSL library available");
+#endif
+}
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
