Changeset: a8afa43ca17e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a8afa43ca17e
Modified Files:
MonetDB.spec
gdk/gdk_bbp.c
monetdb5/mal/mal_profiler.c
sql/storage/bat/bat_storage.c
tools/mserver/mserver5.c
Branch: default
Log Message:
Merge with Oct2020 branch.
diffs (truncated from 413 to 300 lines):
diff --git a/.editorconfig b/.editorconfig
--- a/.editorconfig
+++ b/.editorconfig
@@ -1,3 +1,5 @@
+root = true
+
# Unix-style newlines with a newline ending every file
[*]
end_of_line = lf
@@ -16,7 +18,3 @@ indent_style = space
indent_size = 4
trim_trailing_whitespace = true
charset = utf-8
-
-[gdk/*.{c,h}]
-tab_width = 8
-max_line_length = 72
diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -140,6 +140,13 @@ BuildRequires: pkgconfig(openssl)
BuildRequires: pkgconfig(libpcre) >= 4.5
%endif
BuildRequires: pkgconfig(zlib)
+%if %{?rhel:0}%{!?rhel:1} || 0%{?rhel} > 7
+# not on RHEL 7
+BuildRequires: pkgconfig(liblz4) >= 1.8
+%global LZ4 ON
+%else
+%global LZ4 OFF
+%endif
%if %{with py3integration}
BuildRequires: pkgconfig(python3) >= 3.5
BuildRequires: python3-numpy
@@ -149,8 +156,6 @@ BuildRequires: pkgconfig(libR)
%endif
# if we were to compile with cmocka support (-DWITH_CMOCKA=ON):
# BuildRequires: pkgconfig(cmocka)
-# if we were to compile with lz4 support (-DWITH_LZ4=ON):
-# BuildRequires: pkgconfig(liblz4)
# if we were to compile with NetCDF support (-DNETCDF=ON):
# BuildRequires: pkgconfig(netcdf)
# if we were to compile with proj support (-DWITH_PROJ=ON):
@@ -801,7 +806,7 @@ export CFLAGS
-DWITH_CMOCKA=OFF \
-DWITH_CRYPTO=ON \
-DWITH_CURL=ON \
- -DWITH_LZ4=OFF \
+ -DWITH_LZ4=%{LZ4} \
-DWITH_LZMA=ON \
-DWITH_PCRE=ON \
-DWITH_PROJ=OFF \
diff --git a/debian/control b/debian/control
--- a/debian/control
+++ b/debian/control
@@ -7,7 +7,7 @@ Vcs-Browser: https://dev.monetdb.org/hg/
Vcs-Hg: https://dev.monetdb.org/hg/MonetDB/
Build-Depends: debhelper (>= 9), cmake (>= 3.12), bison,
libbz2-dev, libcurl4-gnutls-dev, libgeos-dev (>= 3.4.0),
- libpcre3-dev, libreadline-dev, liblzma-dev,
+ libpcre3-dev, libreadline-dev, liblzma-dev, liblz4-dev (>= 1.8.0),
libssl-dev, libxml2-dev, pkg-config,
python3, python3-dev, python3-numpy,
unixodbc-dev, uuid-dev, zlib1g-dev, r-base-dev,
diff --git a/debian/rules b/debian/rules
--- a/debian/rules
+++ b/debian/rules
@@ -31,7 +31,7 @@ override_dh_auto_configure:
-DWITH_CMOCKA=OFF \
-DWITH_CRYPTO=ON \
-DWITH_CURL=ON \
- -DWITH_LZ4=OFF \
+ -DWITH_LZ4=ON \
-DWITH_LZMA=ON \
-DWITH_PCRE=ON \
-DWITH_PROJ=OFF \
diff --git a/gdk/.editorconfig b/gdk/.editorconfig
new file mode 100644
--- /dev/null
+++ b/gdk/.editorconfig
@@ -0,0 +1,3 @@
+[*.{c,h}]
+tab_width = 8
+max_line_length = 72
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -3624,6 +3624,10 @@ BBPdiskscan(const char *parent, size_t b
/* older versions used .thash which we
* can simply ignore */
delete = true;
+ } else if (strncmp(p + 1, "thsh", 4) == 0) {
+ /* temporary hash files which we can
+ * simply ignore */
+ delete = true;
} else if (strncmp(p + 1, "timprints", 9) == 0) {
BAT *b = getdesc(bid);
delete = b == NULL;
diff --git a/monetdb5/mal/mal_profiler.c b/monetdb5/mal/mal_profiler.c
--- a/monetdb5/mal/mal_profiler.c
+++ b/monetdb5/mal/mal_profiler.c
@@ -60,17 +60,51 @@ static struct rusage prevUsage;
#define LOGLEN 8192
#define lognew() loglen = 0; logbase = logbuffer; *logbase = 0;
+/*
+ * We use a buffer (`logbuffer`) where we incrementally create the output JSON
object. Initially we allocate LOGLEN (8K)
+ * bytes and we keep the capacity of the buffer (`logcap`) and the length of
the current string (`loglen`).
+ *
+ * We use the `logadd` macro to add data to our buffer (usually key-value
pairs). This macro offers an interface similar
+ * to printf.
+ *
+ * The first snprintf bellow happens in a statically allocated buffer that
might be much smaller than logcap. We do not
+ * care. We only need to perform this snprintf to get the actual length of the
string that is to be produced.
+ *
+ * There are three cases:
+ *
+ * 1. The new string fits in the current buffer -> we just update the buffer
+ *
+ * 2. The new string does not fit in the current buffer, but is smaller than
the capacity of the buffer -> we output the
+ * current contents of the buffer and start at the beginnig.
+ *
+ * 3. The new string exceeds the current capacity of the buffer -> we output
the current contents and reallocate the
+ * buffer. The new capacity is 1.5 times the length of the new string.
+ */
#define logadd(...)
\
do {
\
char tmp_buff[LOGLEN];
\
- int tmp_len = 0;
\
+ size_t tmp_len = 0;
\
tmp_len = snprintf(tmp_buff, LOGLEN, __VA_ARGS__);
\
- if (loglen + tmp_len < LOGLEN)
\
- loglen += snprintf(logbase+loglen, LOGLEN - loglen,
__VA_ARGS__); \
- else {
\
+ if (loglen + tmp_len < logcap)
\
+ loglen += snprintf(logbase+loglen, logcap - loglen,
__VA_ARGS__); \
+ else if (tmp_len < logcap) {
\
logjsonInternal(logbuffer);
\
lognew();
\
- loglen += snprintf(logbase+loglen, LOGLEN - loglen,
__VA_ARGS__); \
+ loglen += snprintf(logbase+loglen, logcap - loglen,
__VA_ARGS__); \
+ }
\
+ else {
\
+ char *alloc_buff;
\
+ logjsonInternal(logbuffer);
\
+ logcap = tmp_len + tmp_len/2;
\
+ alloc_buff = (char *)realloc(logbuffer, logcap);
\
+ if (alloc_buff == NULL) {
\
+ TRC_ERROR(MAL_SERVER, "Profiler JSON buffer
reallocation failure\n"); \
+ free(logbuffer);
\
+ return;
\
+ }
\
+ logbuffer = alloc_buff;
\
+ lognew();
\
+ loglen += snprintf(logbase+loglen, logcap - loglen,
__VA_ARGS__); \
}
\
} while (0)
@@ -89,27 +123,6 @@ static void logjsonInternal(char *logbuf
MT_lock_unset(&mal_profileLock);
}
-static char *
-truncate_string(char *inp)
-{
- size_t len;
- char *ret;
- size_t ret_len = LOGLEN/2;
- size_t padding = 64;
-
- len = strlen(inp);
- ret = (char *)GDKmalloc(ret_len + 1);
- if (ret == NULL) {
- return NULL;
- }
-
- snprintf(ret, ret_len + 1, "%.*s...<truncated>...%.*s",
- (int) (ret_len/2), inp, (int) (ret_len/2 - padding),
- inp + (len - ret_len/2 + padding));
-
- return ret;
-}
-
/* JSON rendering method of performance data.
* The eventparser may assume this layout for ease of parsing
EXAMPLE:
@@ -127,8 +140,8 @@ EXAMPLE:
static void
renderProfilerEvent(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci,
int start)
{
- char logbuffer[LOGLEN], *logbase;
- size_t loglen;
+ char *logbuffer = NULL, *logbase;
+ size_t loglen, logcap = LOGLEN;
str c;
str stmtq;
lng usec;
@@ -150,6 +163,12 @@ renderProfilerEvent(Client cntxt, MalBlk
if(malprofileruser!= MAL_ADMIN && malprofileruser != cntxt->user)
return;
+ logbuffer = (char *)malloc(logcap);
+ if (logbuffer == NULL) {
+ TRC_ERROR(MAL_SERVER, "Profiler JSON buffer allocation
failure\n");
+ return;
+ }
+
usec= pci->clock;
microseconds = (uint64_t)usec - ((uint64_t)startup_time.tv_sec*1000000
- (uint64_t)startup_time.tv_usec);
/* make profile event tuple */
@@ -275,17 +294,11 @@ This information can be used to determin
logadd(",\"count\":"BUNFMT, cnt);
logadd(",\"size\":" LLFMT, total);
} else{
- char *truncated = NULL;
tname = getTypeName(tpe);
logadd(",\"type\":\"%s\"", tname);
logadd(",\"const\":%d",
isVarConstant(mb, getArg(pci,j)));
cv =
VALformat(&stk->stk[getArg(pci,j)]);
stmtq = cv ? mal_quote(cv, strlen(cv))
: NULL;
- if (stmtq != NULL && strlen(stmtq) >
LOGLEN/2) {
- truncated =
truncate_string(stmtq);
- GDKfree(stmtq);
- stmtq = truncated;
- }
if (stmtq)
logadd(",\"value\":\"%s\"",
stmtq);
GDKfree(cv);
@@ -302,6 +315,7 @@ This information can be used to determin
}
logadd("}\n"); // end marker
logjsonInternal(logbuffer);
+ free(logbuffer);
}
/* the OS details on cpu load are read from /proc/stat
@@ -381,8 +395,8 @@ void
profilerHeartbeatEvent(char *alter)
{
char cpuload[BUFSIZ];
- char logbuffer[LOGLEN], *logbase;
- int loglen;
+ char *logbuffer = NULL, *logbase;
+ size_t loglen, logcap = LOGLEN;
lng usec;
uint64_t microseconds;
@@ -395,6 +409,12 @@ profilerHeartbeatEvent(char *alter)
if (getCPULoad(cpuload))
return;
+ logbuffer = (char *)malloc(logcap);
+ if (logbuffer == NULL) {
+ TRC_ERROR(MAL_SERVER, "Profiler JSON buffer allocation
failure\n");
+ return;
+ }
+
lognew();
logadd("{"); // fill in later with the event counter
if (!GDKinmemory(0) && !GDKembedded()) {
@@ -426,6 +446,7 @@ profilerHeartbeatEvent(char *alter)
logadd("\"cpuload\":%s",cpuload);
logadd("}\n"); // end marker
logjsonInternal(logbuffer);
+ free(logbuffer);
}
void
diff --git a/monetdb5/optimizer/opt_mergetable.c
b/monetdb5/optimizer/opt_mergetable.c
--- a/monetdb5/optimizer/opt_mergetable.c
+++ b/monetdb5/optimizer/opt_mergetable.c
@@ -59,15 +59,6 @@ is_a_mat(int idx, matlist_t *ml)
}
static int
-was_a_mat(int idx, matlist_t *ml){
- int i;
- for(i =0; i<ml->top; i++)
- if (ml->v[i].mv == idx)
- return i;
- return -1;
-}
-
-static int
nr_of_mats(InstrPtr p, matlist_t *ml)
{
int j,cnt=0;
@@ -82,7 +73,7 @@ nr_of_bats(MalBlkPtr mb, InstrPtr p)
{
int j,cnt=0;
for(j=p->retc; j<p->argc; j++)
- if (isaBatType(getArgType(mb,p,j)))
+ if (isaBatType(getArgType(mb,p,j)) && !isVarConstant(mb,
getArg(p,j)))
cnt++;
return cnt;
}
@@ -92,7 +83,7 @@ nr_of_nilbats(MalBlkPtr mb, InstrPtr p)
{
int j,cnt=0;
for(j=p->retc; j<p->argc; j++)
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
