Hello community, here is the log from the commit of package redis for openSUSE:Factory checked in at 2018-04-13 12:51:17 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/redis (Old) and /work/SRC/openSUSE:Factory/.redis.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "redis" Fri Apr 13 12:51:17 2018 rev:40 rq:595984 version:4.0.9 Changes: -------- --- /work/SRC/openSUSE:Factory/redis/redis.changes 2018-02-09 15:52:54.871822767 +0100 +++ /work/SRC/openSUSE:Factory/.redis.new/redis.changes 2018-04-13 12:51:47.935142369 +0200 @@ -1,0 +2,17 @@ +Thu Apr 12 13:13:49 UTC 2018 - [email protected] + +- Update to 4.0.9 + * https://raw.githubusercontent.com/antirez/redis/4.0/00-RELEASENOTES + * Critical upgrade for users using AOF with the fsync policy set to "always". + * Latency monitor could report wrong latencies under certain conditions. + * AOF rewriting could fail when a backgronud rewrite is triggered and + at the same time the AOF is switched on/off. + * Redis Cluster crash-recovery safety improved. + * Redis Cluster has now the ability to configure certain slaves so that + they'll never attempt a failover. + * Keyspace notifications API in modules. + * RM_Call() is now faster by reusing the same client. + * Tracking of the percentage of keys already logically expired but yet + not evicted. + +------------------------------------------------------------------- Old: ---- redis-4.0.8.tar.gz New: ---- redis-4.0.9.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ redis.spec ++++++ --- /var/tmp/diff_new_pack.1Hq4nV/_old 2018-04-13 12:51:49.139098749 +0200 +++ /var/tmp/diff_new_pack.1Hq4nV/_new 2018-04-13 12:51:49.139098749 +0200 @@ -19,7 +19,7 @@ %define _log_dir %{_localstatedir}/log/%{name} %define _conf_dir %{_sysconfdir}/%{name} Name: redis -Version: 4.0.8 +Version: 4.0.9 Release: 0 Summary: Persistent key-value database License: BSD-3-Clause ++++++ redis-4.0.8.tar.gz -> redis-4.0.9.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/00-RELEASENOTES new/redis-4.0.9/00-RELEASENOTES --- old/redis-4.0.8/00-RELEASENOTES 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/00-RELEASENOTES 2018-03-26 18:04:15.000000000 +0200 @@ -11,6 +11,199 @@ -------------------------------------------------------------------------------- ================================================================================ +Redis 4.0.9 Released Mon Mar 26 17:52:32 CEST 2018 +================================================================================ + +Upgrade urgency CRITICAL: Critical upgrade for users using AOF with the + fsync policy set to "always". + +Dear Redis users, + +Redis version 4.0.9 adds a few interesting new features and fixes a very +critical bug regarding the Append Only File. Let's start with the bad news +(the critical bug), explaining what happens and in what conditions: + +Critical AOF bug explained +-------------------------- + +When AOF is enabled with the fsync policy set to "always", we have a +(rarely used) setup where Redis fsyncs every new write on disk. On this +setup Redis MUST reply to the client with an OK code to the write, only +after the write is already persisted on disk. + +Because of a bug, in particular conditions, it sometimes happens (verified +experimentally that the condition can be actually created) that in the +same event loop cycle the command is both processed and the reply sent, before +the beforeSleep() function has the ability to fsync the write on disk. + +The redis 4.0.9 release fixes this problem introducing the concept of +write barriers in the Redis event loop (ae.c). If you are using a different +AOF setup, like fsync everysec, you are not affected because such guarantee +is not provided anyway. Similarly if you have fsync set to always but you +do not semantically use the fact that the reply is only sent after the +successful fsync, you may avoid upgrading. + +Other bugfixes +-------------- + +Other things that we fixed in this release include: + +* Latency monitor could report wrong latencies under certain conditions. +* AOF rewriting could fail when a backgronud rewrite is triggered and + at the same time the AOF is switched on/off. +* Redis Cluster crash-recovery safety improved. +* Other smaller fixes (check commnits). + +New features +------------ + +* Redis Cluster has now the ability to configure certain slaves so that + they'll never attempt a failover. +* Keyspace notifications API in modules. +* RM_Call() is now faster by reusing the same client. +* Tracking of the percentage of keys already logically expired but yet + not evicted. +* Other smaller improvements (check commits) + +This is the list of commits composing this release: + +zhaozhao.zz in commit 5b722bd7: + fix missed call on freeaddrinfo + 1 file changed, 1 insertion(+), 1 deletion(-) + +zhaozhao.zz in commit 2551b0f6: + anet: avoid double close + 1 file changed, 1 insertion(+), 1 deletion(-) + +antirez in commit 8d92885b: + Cluster: add test for the nofailover flag. + 2 files changed, 71 insertions(+) + +antirez in commit 70597a30: + Cluster: ability to prevent slaves from failing over their masters. + 6 files changed, 70 insertions(+), 2 deletions(-) + +antirez in commit 16cad10a: + redis-cli: fix missed unit in array. Change define name. + 1 file changed, 5 insertions(+), 5 deletions(-) + +charsyam in commit 640fa434: + fix-out-of-index-range-for-redis-cli-findbigkey + 1 file changed, 6 insertions(+), 4 deletions(-) + +antirez in commit 83390f55: + expireIfNeeded() needed a top comment documenting the behavior. + 1 file changed, 19 insertions(+) + +antirez in commit 888039ca: + expireIfNeeded() comment: claim -> pretend. + 1 file changed, 1 insertion(+), 1 deletion(-) + +antirez in commit e09c8c10: + Actually use ae_flags to add AE_BARRIER if needed. + 1 file changed, 1 insertion(+), 1 deletion(-) + +charsyam in commit fb7560bc: + refactoring-make-condition-clear-for-rdb + 1 file changed, 2 insertions(+), 2 deletions(-) + +antirez in commit 1e2f0d69: + ae.c: insetad of not firing, on AE_BARRIER invert the sequence. + 1 file changed, 38 insertions(+), 22 deletions(-) + +antirez in commit b2e4aad9: + AOF: fix a bug that may prevent proper fsyncing when fsync=always. + 1 file changed, 18 insertions(+), 6 deletions(-) + +antirez in commit 93bad8ae: + Cluster: improve crash-recovery safety after failover auth vote. + 1 file changed, 3 insertions(+), 2 deletions(-) + +antirez in commit e32752e8: + ae.c: introduce the concept of read->write barrier. + 2 files changed, 29 insertions(+), 6 deletions(-) + +antirez in commit 262f4039: + Fix ziplist prevlen encoding description. See #4705. + 1 file changed, 6 insertions(+), 6 deletions(-) + +antirez in commit 83923afa: + Track number of logically expired keys still in memory. + 3 files changed, 28 insertions(+), 1 deletion(-) + +antirez in commit 256ddbf6: + Remove non semantical spaces from module.c. + 1 file changed, 36 insertions(+), 41 deletions(-) + +antirez in commit 280c3e39: + Fix typo in notifyKeyspaceEvent() comment. + 1 file changed, 1 insertion(+), 1 deletion(-) + +Dvir Volk in commit 7c4623b0: + Add doc comment about notification flags + 1 file changed, 1 insertion(+) + +Dvir Volk in commit f4e7502e: + Fix indentation and comment style in testmodule + 1 file changed, 92 insertions(+), 98 deletions(-) + +Dvir Volk in commit 3c8456c6: + Use one static client for all keyspace notification callbacks + 1 file changed, 11 insertions(+), 7 deletions(-) + +Dvir Volk in commit aaaff8bd: + Remove the NOTIFY_MODULE flag and simplify the module notification flow if there aren't subscribers + 3 files changed, 5 insertions(+), 9 deletions(-) + +Dvir Volk in commit 0be51b8f: + Document flags for notifications + 1 file changed, 17 insertions(+), 1 deletion(-) + +Dvir Volk in commit 3b95c89c: + removed some trailing whitespaces + 1 file changed, 2 deletions(-) + +Dvir Volk in commit 84c6f1e3: + removed hellonotify.c + 3 files changed, 1 insertion(+), 87 deletions(-) + +Dvir Volk in commit 53b85e53: + fixed test + 1 file changed, 7 insertions(+), 1 deletion(-) + +Dvir Volk in commit b43f66c9: + finished implementation of notifications. Tests unfinished + 7 files changed, 339 insertions(+), 3 deletions(-) + +antirez in commit eddf5deb: + More verbose logging when slave sends errors to master. + 1 file changed, 6 insertions(+), 2 deletions(-) + +oranagra in commit c09cc0a9: + when a slave experiances an error on commands that come from master, print to the log + 1 file changed, 2 insertions(+) + +charsyam in commit 5c374f94: + getting rid of duplicated code + 1 file changed, 2 insertions(+), 2 deletions(-) + +Guy Benoish in commit a64f36e5: + enlarged buffer given to ld2string + 3 files changed, 7 insertions(+), 2 deletions(-) + +antirez in commit f1705801: + Make it explicit with a comment why we kill the old AOF rewrite. + 1 file changed, 3 insertions(+) + +Guy Benoish in commit 0c030dea: + rewriteAppendOnlyFileBackground() failure fix + 1 file changed, 31 insertions(+), 21 deletions(-) + +Oran Agra in commit 58073974: + fix to latency monitor reporting wrong max latency + 1 file changed, 1 insertion(+) + +================================================================================ Redis 4.0.8 Released Fri Feb 2 11:17:40 CET 2018 ================================================================================ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/redis.conf new/redis-4.0.9/redis.conf --- old/redis-4.0.8/redis.conf 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/redis.conf 2018-03-26 18:04:15.000000000 +0200 @@ -904,6 +904,16 @@ # # cluster-require-full-coverage yes +# This option, when set to yes, prevents slaves from trying to failover its +# master during master failures. However the master can still perform a +# manual failover, if forced to do so. +# +# This is useful in different scenarios, especially in the case of multiple +# data center operations, where we want one side to never be promoted if not +# in the case of a total DC failure. +# +# cluster-slave-no-failover no + # In order to setup your cluster make sure to read the documentation # available at http://redis.io web site. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/ae.c new/redis-4.0.9/src/ae.c --- old/redis-4.0.8/src/ae.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/ae.c 2018-03-26 18:04:15.000000000 +0200 @@ -159,6 +159,10 @@ aeFileEvent *fe = &eventLoop->events[fd]; if (fe->mask == AE_NONE) return; + /* We want to always remove AE_BARRIER if set when AE_WRITABLE + * is removed. */ + if (mask & AE_WRITABLE) mask |= AE_BARRIER; + aeApiDelEvent(eventLoop, fd, mask); fe->mask = fe->mask & (~mask); if (fd == eventLoop->maxfd && fe->mask == AE_NONE) { @@ -411,19 +415,49 @@ aeFileEvent *fe = &eventLoop->events[eventLoop->fired[j].fd]; int mask = eventLoop->fired[j].mask; int fd = eventLoop->fired[j].fd; - int rfired = 0; + int fired = 0; /* Number of events fired for current fd. */ - /* note the fe->mask & mask & ... code: maybe an already processed - * event removed an element that fired and we still didn't - * processed, so we check if the event is still valid. */ - if (fe->mask & mask & AE_READABLE) { - rfired = 1; + /* Normally we execute the readable event first, and the writable + * event laster. This is useful as sometimes we may be able + * to serve the reply of a query immediately after processing the + * query. + * + * However if AE_BARRIER is set in the mask, our application is + * asking us to do the reverse: never fire the writable event + * after the readable. In such a case, we invert the calls. + * This is useful when, for instance, we want to do things + * in the beforeSleep() hook, like fsynching a file to disk, + * before replying to a client. */ + int invert = fe->mask & AE_BARRIER; + + /* Note the "fe->mask & mask & ..." code: maybe an already + * processed event removed an element that fired and we still + * didn't processed, so we check if the event is still valid. + * + * Fire the readable event if the call sequence is not + * inverted. */ + if (!invert && fe->mask & mask & AE_READABLE) { fe->rfileProc(eventLoop,fd,fe->clientData,mask); + fired++; } + + /* Fire the writable event. */ if (fe->mask & mask & AE_WRITABLE) { - if (!rfired || fe->wfileProc != fe->rfileProc) + if (!fired || fe->wfileProc != fe->rfileProc) { fe->wfileProc(eventLoop,fd,fe->clientData,mask); + fired++; + } } + + /* If we have to invert the call, fire the readable event now + * after the writable one. */ + if (invert && fe->mask & mask & AE_READABLE) { + if (!fired || fe->wfileProc != fe->rfileProc) { + fe->rfileProc(eventLoop,fd,fe->clientData,mask); + fired++; + } + } + processed++; } } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/ae.h new/redis-4.0.9/src/ae.h --- old/redis-4.0.8/src/ae.h 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/ae.h 2018-03-26 18:04:15.000000000 +0200 @@ -38,9 +38,14 @@ #define AE_OK 0 #define AE_ERR -1 -#define AE_NONE 0 -#define AE_READABLE 1 -#define AE_WRITABLE 2 +#define AE_NONE 0 /* No events registered. */ +#define AE_READABLE 1 /* Fire when descriptor is readable. */ +#define AE_WRITABLE 2 /* Fire when descriptor is writable. */ +#define AE_BARRIER 4 /* With WRITABLE, never fire the event if the + READABLE event already fired in the same event + loop iteration. Useful when you want to persist + things to disk before sending replies, and want + to do that in a group fashion. */ #define AE_FILE_EVENTS 1 #define AE_TIME_EVENTS 2 @@ -64,7 +69,7 @@ /* File event structure */ typedef struct aeFileEvent { - int mask; /* one of AE_(READABLE|WRITABLE) */ + int mask; /* one of AE_(READABLE|WRITABLE|BARRIER) */ aeFileProc *rfileProc; aeFileProc *wfileProc; void *clientData; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/anet.c new/redis-4.0.9/src/anet.c --- old/redis-4.0.8/src/anet.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/anet.c 2018-03-26 18:04:15.000000000 +0200 @@ -484,7 +484,7 @@ if (af == AF_INET6 && anetV6Only(err,s) == ANET_ERR) goto error; if (anetSetReuseAddr(err,s) == ANET_ERR) goto error; - if (anetListen(err,s,p->ai_addr,p->ai_addrlen,backlog) == ANET_ERR) goto error; + if (anetListen(err,s,p->ai_addr,p->ai_addrlen,backlog) == ANET_ERR) s = ANET_ERR; goto end; } if (p == NULL) { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/aof.c new/redis-4.0.9/src/aof.c --- old/redis-4.0.8/src/aof.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/aof.c 2018-03-26 18:04:15.000000000 +0200 @@ -203,6 +203,26 @@ bioCreateBackgroundJob(BIO_AOF_FSYNC,(void*)(long)fd,NULL,NULL); } +/* Kills an AOFRW child process if exists */ +static void killAppendOnlyChild(void) { + int statloc; + /* No AOFRW child? return. */ + if (server.aof_child_pid == -1) return; + /* Kill AOFRW child, wait for child exit. */ + serverLog(LL_NOTICE,"Killing running AOF rewrite child: %ld", + (long) server.aof_child_pid); + if (kill(server.aof_child_pid,SIGUSR1) != -1) { + while(wait3(&statloc,0,NULL) != server.aof_child_pid); + } + /* Reset the buffer accumulating changes while the child saves. */ + aofRewriteBufferReset(); + aofRemoveTempFile(server.aof_child_pid); + server.aof_child_pid = -1; + server.aof_rewrite_time_start = -1; + /* Close pipes used for IPC between the two processes. */ + aofClosePipes(); +} + /* Called when the user switches from "appendonly yes" to "appendonly no" * at runtime using the CONFIG command. */ void stopAppendOnly(void) { @@ -214,23 +234,7 @@ server.aof_fd = -1; server.aof_selected_db = -1; server.aof_state = AOF_OFF; - /* rewrite operation in progress? kill it, wait child exit */ - if (server.aof_child_pid != -1) { - int statloc; - - serverLog(LL_NOTICE,"Killing running AOF rewrite child: %ld", - (long) server.aof_child_pid); - if (kill(server.aof_child_pid,SIGUSR1) != -1) { - while(wait3(&statloc,0,NULL) != server.aof_child_pid); - } - /* reset the buffer accumulating changes while the child saves */ - aofRewriteBufferReset(); - aofRemoveTempFile(server.aof_child_pid); - server.aof_child_pid = -1; - server.aof_rewrite_time_start = -1; - /* close pipes used for IPC between the two processes. */ - aofClosePipes(); - } + killAppendOnlyChild(); } /* Called when the user switches from "appendonly no" to "appendonly yes" @@ -255,10 +259,19 @@ if (server.rdb_child_pid != -1) { server.aof_rewrite_scheduled = 1; serverLog(LL_WARNING,"AOF was enabled but there is already a child process saving an RDB file on disk. An AOF background was scheduled to start when possible."); - } else if (rewriteAppendOnlyFileBackground() == C_ERR) { - close(newfd); - serverLog(LL_WARNING,"Redis needs to enable the AOF but can't trigger a background AOF rewrite operation. Check the above logs for more info about the error."); - return C_ERR; + } else { + /* If there is a pending AOF rewrite, we need to switch it off and + * start a new one: the old one cannot be reused becuase it is not + * accumulating the AOF buffer. */ + if (server.aof_child_pid != -1) { + serverLog(LL_WARNING,"AOF was enabled but there is already an AOF rewriting in background. Stopping background AOF and starting a rewrite now."); + killAppendOnlyChild(); + } + if (rewriteAppendOnlyFileBackground() == C_ERR) { + close(newfd); + serverLog(LL_WARNING,"Redis needs to enable the AOF but can't trigger a background AOF rewrite operation. Check the above logs for more info about the error."); + return C_ERR; + } } /* We correctly switched on AOF, now wait for the rewrite to be complete * in order to append data on disk. */ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/cluster.c new/redis-4.0.9/src/cluster.c --- old/redis-4.0.8/src/cluster.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/cluster.c 2018-03-26 18:04:15.000000000 +0200 @@ -201,6 +201,8 @@ n->flags |= CLUSTER_NODE_HANDSHAKE; } else if (!strcasecmp(s,"noaddr")) { n->flags |= CLUSTER_NODE_NOADDR; + } else if (!strcasecmp(s,"nofailover")) { + n->flags |= CLUSTER_NODE_NOFAILOVER; } else if (!strcasecmp(s,"noflags")) { /* nothing to do */ } else { @@ -407,6 +409,22 @@ return C_OK; } +/* Some flags (currently just the NOFAILOVER flag) may need to be updated + * in the "myself" node based on the current configuration of the node, + * that may change at runtime via CONFIG SET. This function changes the + * set of flags in myself->flags accordingly. */ +void clusterUpdateMyselfFlags(void) { + int oldflags = myself->flags; + int nofailover = server.cluster_slave_no_failover ? + CLUSTER_NODE_NOFAILOVER : 0; + myself->flags &= ~CLUSTER_NODE_NOFAILOVER; + myself->flags |= nofailover; + if (myself->flags != oldflags) { + clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG| + CLUSTER_TODO_UPDATE_STATE); + } +} + void clusterInit(void) { int saveconf = 0; @@ -497,6 +515,7 @@ server.cluster->mf_end = 0; resetManualFailover(); + clusterUpdateMyselfFlags(); } /* Reset a node performing a soft or hard reset: @@ -1808,6 +1827,18 @@ } } + /* Copy the CLUSTER_NODE_NOFAILOVER flag from what the sender + * announced. This is a dynamic flag that we receive from the + * sender, and the latest status must be trusted. We need it to + * be propagated because the slave ranking used to understand the + * delay of each slave in the voting process, needs to know + * what are the instances really competing. */ + if (sender) { + int nofailover = flags & CLUSTER_NODE_NOFAILOVER; + sender->flags &= ~CLUSTER_NODE_NOFAILOVER; + sender->flags |= nofailover; + } + /* Update the node address if it changed. */ if (sender && type == CLUSTERMSG_TYPE_PING && !nodeInHandshake(sender) && @@ -2156,7 +2187,7 @@ * from event handlers that will do stuff with the same link later. */ void clusterSendMessage(clusterLink *link, unsigned char *msg, size_t msglen) { if (sdslen(link->sndbuf) == 0 && msglen != 0) - aeCreateFileEvent(server.el,link->fd,AE_WRITABLE, + aeCreateFileEvent(server.el,link->fd,AE_WRITABLE|AE_BARRIER, clusterWriteHandler,link); link->sndbuf = sdscatlen(link->sndbuf, msg, msglen); @@ -2691,9 +2722,10 @@ } /* We can vote for this slave. */ - clusterSendFailoverAuth(node); server.cluster->lastVoteEpoch = server.cluster->currentEpoch; node->slaveof->voted_time = mstime(); + clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|CLUSTER_TODO_FSYNC_CONFIG); + clusterSendFailoverAuth(node); serverLog(LL_WARNING, "Failover auth granted to %.40s for epoch %llu", node->name, (unsigned long long) server.cluster->currentEpoch); } @@ -2722,6 +2754,7 @@ myoffset = replicationGetSlaveOffset(); for (j = 0; j < master->numslaves; j++) if (master->slaves[j] != myself && + !nodeCantFailover(master->slaves[j]) && master->slaves[j]->repl_offset > myoffset) rank++; return rank; } @@ -2859,10 +2892,13 @@ * of an automatic or manual failover: * 1) We are a slave. * 2) Our master is flagged as FAIL, or this is a manual failover. - * 3) It is serving slots. */ + * 3) We don't have the no failover configuration set, and this is + * not a manual failover. + * 4) It is serving slots. */ if (nodeIsMaster(myself) || myself->slaveof == NULL || (!nodeFailed(myself->slaveof) && !manual_failover) || + (server.cluster_slave_no_failover && !manual_failover) || myself->slaveof->numslots == 0) { /* There are no reasons to failover, so we set the reason why we @@ -3238,6 +3274,9 @@ handshake_timeout = server.cluster_node_timeout; if (handshake_timeout < 1000) handshake_timeout = 1000; + /* Update myself flags. */ + clusterUpdateMyselfFlags(); + /* Check if we have disconnected nodes and re-establish the connection. * Also update a few stats while we are here, that can be used to make * better decisions in other part of the code. */ @@ -3836,7 +3875,8 @@ {CLUSTER_NODE_PFAIL, "fail?,"}, {CLUSTER_NODE_FAIL, "fail,"}, {CLUSTER_NODE_HANDSHAKE, "handshake,"}, - {CLUSTER_NODE_NOADDR, "noaddr,"} + {CLUSTER_NODE_NOADDR, "noaddr,"}, + {CLUSTER_NODE_NOFAILOVER, "nofailover,"} }; /* Concatenate the comma separated list of node flags to the given SDS diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/cluster.h new/redis-4.0.9/src/cluster.h --- old/redis-4.0.8/src/cluster.h 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/cluster.h 2018-03-26 18:04:15.000000000 +0200 @@ -16,6 +16,7 @@ #define CLUSTER_DEFAULT_NODE_TIMEOUT 15000 #define CLUSTER_DEFAULT_SLAVE_VALIDITY 10 /* Slave max data age factor. */ #define CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE 1 +#define CLUSTER_DEFAULT_SLAVE_NO_FAILOVER 0 /* Failover by default. */ #define CLUSTER_FAIL_REPORT_VALIDITY_MULT 2 /* Fail report validity. */ #define CLUSTER_FAIL_UNDO_TIME_MULT 2 /* Undo fail if master is back. */ #define CLUSTER_FAIL_UNDO_TIME_ADD 10 /* Some additional time. */ @@ -55,6 +56,7 @@ #define CLUSTER_NODE_NOADDR 64 /* We don't know the address of this node */ #define CLUSTER_NODE_MEET 128 /* Send a MEET message to this node */ #define CLUSTER_NODE_MIGRATE_TO 256 /* Master elegible for replica migration. */ +#define CLUSTER_NODE_NOFAILOVER 512 /* Slave will not try to failver. */ #define CLUSTER_NODE_NULL_NAME "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" #define nodeIsMaster(n) ((n)->flags & CLUSTER_NODE_MASTER) @@ -64,6 +66,7 @@ #define nodeWithoutAddr(n) ((n)->flags & CLUSTER_NODE_NOADDR) #define nodeTimedOut(n) ((n)->flags & CLUSTER_NODE_PFAIL) #define nodeFailed(n) ((n)->flags & CLUSTER_NODE_FAIL) +#define nodeCantFailover(n) ((n)->flags & CLUSTER_NODE_NOFAILOVER) /* Reasons why a slave is not able to failover. */ #define CLUSTER_CANT_FAILOVER_NONE 0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/config.c new/redis-4.0.9/src/config.c --- old/redis-4.0.8/src/config.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/config.c 2018-03-26 18:04:15.000000000 +0200 @@ -635,6 +635,14 @@ err = "cluster slave validity factor must be zero or positive"; goto loaderr; } + } else if (!strcasecmp(argv[0],"cluster-slave-no-failover") && + argc == 2) + { + server.cluster_slave_no_failover = yesnotoi(argv[1]); + if (server.cluster_slave_no_failover == -1) { + err = "argument must be 'yes' or 'no'"; + goto loaderr; + } } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { server.lua_time_limit = strtoll(argv[1],NULL,10); } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && @@ -998,6 +1006,8 @@ } config_set_bool_field( "cluster-require-full-coverage",server.cluster_require_full_coverage) { } config_set_bool_field( + "cluster-slave-no-failover",server.cluster_slave_no_failover) { + } config_set_bool_field( "aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync) { } config_set_bool_field( "aof-load-truncated",server.aof_load_truncated) { @@ -1291,6 +1301,8 @@ /* Bool (yes/no) values */ config_get_bool_field("cluster-require-full-coverage", server.cluster_require_full_coverage); + config_get_bool_field("cluster-slave-no-failover", + server.cluster_slave_no_failover); config_get_bool_field("no-appendfsync-on-rewrite", server.aof_no_fsync_on_rewrite); config_get_bool_field("slave-serve-stale-data", @@ -2023,6 +2035,7 @@ rewriteConfigYesNoOption(state,"cluster-enabled",server.cluster_enabled,0); rewriteConfigStringOption(state,"cluster-config-file",server.cluster_configfile,CONFIG_DEFAULT_CLUSTER_CONFIG_FILE); rewriteConfigYesNoOption(state,"cluster-require-full-coverage",server.cluster_require_full_coverage,CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE); + rewriteConfigYesNoOption(state,"cluster-slave-no-failover",server.cluster_slave_no_failover,CLUSTER_DEFAULT_SLAVE_NO_FAILOVER); rewriteConfigNumericalOption(state,"cluster-node-timeout",server.cluster_node_timeout,CLUSTER_DEFAULT_NODE_TIMEOUT); rewriteConfigNumericalOption(state,"cluster-migration-barrier",server.cluster_migration_barrier,CLUSTER_DEFAULT_MIGRATION_BARRIER); rewriteConfigNumericalOption(state,"cluster-slave-validity-factor",server.cluster_slave_validity_factor,CLUSTER_DEFAULT_SLAVE_VALIDITY); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/db.c new/redis-4.0.9/src/db.c --- old/redis-4.0.8/src/db.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/db.c 2018-03-26 18:04:15.000000000 +0200 @@ -1094,6 +1094,25 @@ decrRefCount(argv[1]); } +/* This function is called when we are going to perform some operation + * in a given key, but such key may be already logically expired even if + * it still exists in the database. The main way this function is called + * is via lookupKey*() family of functions. + * + * The behavior of the function depends on the replication role of the + * instance, because slave instances do not expire keys, they wait + * for DELs from the master for consistency matters. However even + * slaves will try to have a coherent return value for the function, + * so that read commands executed in the slave side will be able to + * behave like if the key is expired even if still present (because the + * master has yet to propagate the DEL). + * + * In masters as a side effect of finding a key which is expired, such + * key will be evicted from the database. Also this may trigger the + * propagation of a DEL/UNLINK command in AOF / replication stream. + * + * The return value of the function is 0 if the key is still valid, + * otherwise the function returns 1 if the key is expired. */ int expireIfNeeded(redisDb *db, robj *key) { mstime_t when = getExpire(db,key); mstime_t now; @@ -1103,7 +1122,7 @@ /* Don't expire anything while loading. It will be done later. */ if (server.loading) return 0; - /* If we are in the context of a Lua script, we claim that time is + /* If we are in the context of a Lua script, we pretend that time is * blocked to when the Lua script started. This way a key can expire * only the first time it is accessed and not in the middle of the * script execution, making propagation to slaves / AOF consistent. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/expire.c new/redis-4.0.9/src/expire.c --- old/redis-4.0.8/src/expire.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/expire.c 2018-03-26 18:04:15.000000000 +0200 @@ -111,7 +111,7 @@ if (clientsArePaused()) return; if (type == ACTIVE_EXPIRE_CYCLE_FAST) { - /* Don't start a fast cycle if the previous cycle did not exited + /* Don't start a fast cycle if the previous cycle did not exit * for time limt. Also don't repeat a fast cycle for the same period * as the fast cycle total duration itself. */ if (!timelimit_exit) return; @@ -140,6 +140,12 @@ if (type == ACTIVE_EXPIRE_CYCLE_FAST) timelimit = ACTIVE_EXPIRE_CYCLE_FAST_DURATION; /* in microseconds. */ + /* Accumulate some global stats as we expire keys, to have some idea + * about the number of keys that are already logically expired, but still + * existing inside the database. */ + long total_sampled = 0; + long total_expired = 0; + for (j = 0; j < dbs_per_call && timelimit_exit == 0; j++) { int expired; redisDb *db = server.db+(current_db % server.dbnum); @@ -192,7 +198,9 @@ ttl_sum += ttl; ttl_samples++; } + total_sampled++; } + total_expired += expired; /* Update the average TTL stats for this database. */ if (ttl_samples) { @@ -212,6 +220,7 @@ elapsed = ustime()-start; if (elapsed > timelimit) { timelimit_exit = 1; + server.stat_expired_time_cap_reached_count++; break; } } @@ -222,6 +231,16 @@ elapsed = ustime()-start; latencyAddSampleIfNeeded("expire-cycle",elapsed/1000); + + /* Update our estimate of keys existing but yet to be expired. + * Running average with this sample accounting for 5%. */ + double current_perc; + if (total_sampled) { + current_perc = (double)total_expired/total_sampled; + } else + current_perc = 0; + server.stat_expired_stale_perc = (current_perc*0.05)+ + (server.stat_expired_stale_perc*0.95); } /*----------------------------------------------------------------------------- diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/latency.c new/redis-4.0.9/src/latency.c --- old/redis-4.0.8/src/latency.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/latency.c 2018-03-26 18:04:15.000000000 +0200 @@ -109,6 +109,8 @@ dictAdd(server.latency_events,zstrdup(event),ts); } + if (latency > ts->max) ts->max = latency; + /* If the previous sample is in the same second, we update our old sample * if this latency is > of the old one, or just return. */ prev = (ts->idx + LATENCY_TS_LEN - 1) % LATENCY_TS_LEN; @@ -120,7 +122,6 @@ ts->samples[ts->idx].time = time(NULL); ts->samples[ts->idx].latency = latency; - if (latency > ts->max) ts->max = latency; ts->idx++; if (ts->idx == LATENCY_TS_LEN) ts->idx = 0; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/module.c new/redis-4.0.9/src/module.c --- old/redis-4.0.8/src/module.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/module.c 2018-03-26 18:04:15.000000000 +0200 @@ -216,6 +216,31 @@ * allow thread safe contexts to execute commands at a safe moment. */ static pthread_mutex_t moduleGIL = PTHREAD_MUTEX_INITIALIZER; + +/* Function pointer type for keyspace event notification subscriptions from modules. */ +typedef int (*RedisModuleNotificationFunc) (RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key); + +/* Keyspace notification subscriber information. + * See RM_SubscribeToKeyspaceEvents() for more information. */ +typedef struct RedisModuleKeyspaceSubscriber { + /* The module subscribed to the event */ + RedisModule *module; + /* Notification callback in the module*/ + RedisModuleNotificationFunc notify_callback; + /* A bit mask of the events the module is interested in */ + int event_mask; + /* Active flag set on entry, to avoid reentrant subscribers + * calling themselves */ + int active; +} RedisModuleKeyspaceSubscriber; + +/* The module keyspace notification subscribers list */ +static list *moduleKeyspaceSubscribers; + +/* Static client recycled for all notification clients, to avoid allocating + * per round. */ +static client *moduleKeyspaceSubscribersClient; + /* -------------------------------------------------------------------------- * Prototypes * -------------------------------------------------------------------------- */ @@ -3669,6 +3694,120 @@ pthread_mutex_unlock(&moduleGIL); } + +/* -------------------------------------------------------------------------- + * Module Keyspace Notifications API + * -------------------------------------------------------------------------- */ + +/* Subscribe to keyspace notifications. This is a low-level version of the + * keyspace-notifications API. A module cand register callbacks to be notified + * when keyspce events occur. + * + * Notification events are filtered by their type (string events, set events, + * etc), and the subsriber callback receives only events that match a specific + * mask of event types. + * + * When subscribing to notifications with RedisModule_SubscribeToKeyspaceEvents + * the module must provide an event type-mask, denoting the events the subscriber + * is interested in. This can be an ORed mask of any of the following flags: + * + * - REDISMODULE_NOTIFY_GENERIC: Generic commands like DEL, EXPIRE, RENAME + * - REDISMODULE_NOTIFY_STRING: String events + * - REDISMODULE_NOTIFY_LIST: List events + * - REDISMODULE_NOTIFY_SET: Set events + * - REDISMODULE_NOTIFY_HASH: Hash events + * - REDISMODULE_NOTIFY_ZSET: Sorted Set events + * - REDISMODULE_NOTIFY_EXPIRED: Expiration events + * - REDISMODULE_NOTIFY_EVICTED: Eviction events + * - REDISMODULE_NOTIFY_STREAM: Stream events + * - REDISMODULE_NOTIFY_ALL: All events + * + * We do not distinguish between key events and keyspace events, and it is up + * to the module to filter the actions taken based on the key. + * + * The subscriber signature is: + * + * int (*RedisModuleNotificationFunc) (RedisModuleCtx *ctx, int type, + * const char *event, + * RedisModuleString *key); + * + * `type` is the event type bit, that must match the mask given at registration + * time. The event string is the actual command being executed, and key is the + * relevant Redis key. + * + * Notification callback gets executed with a redis context that can not be + * used to send anything to the client, and has the db number where the event + * occured as its selected db number. + * + * Notice that it is not necessary to enable norifications in redis.conf for + * module notifications to work. + * + * Warning: the notification callbacks are performed in a synchronous manner, + * so notification callbacks must to be fast, or they would slow Redis down. + * If you need to take long actions, use threads to offload them. + * + * See https://redis.io/topics/notifications for more information. + */ +int RM_SubscribeToKeyspaceEvents(RedisModuleCtx *ctx, int types, RedisModuleNotificationFunc callback) { + RedisModuleKeyspaceSubscriber *sub = zmalloc(sizeof(*sub)); + sub->module = ctx->module; + sub->event_mask = types; + sub->notify_callback = callback; + sub->active = 0; + + listAddNodeTail(moduleKeyspaceSubscribers, sub); + return REDISMODULE_OK; +} + +/* Dispatcher for keyspace notifications to module subscriber functions. + * This gets called only if at least one module requested to be notified on + * keyspace notifications */ +void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid) { + /* Don't do anything if there aren't any subscribers */ + if (listLength(moduleKeyspaceSubscribers) == 0) return; + + listIter li; + listNode *ln; + listRewind(moduleKeyspaceSubscribers,&li); + + /* Remove irrelevant flags from the type mask */ + type &= ~(NOTIFY_KEYEVENT | NOTIFY_KEYSPACE); + + while((ln = listNext(&li))) { + RedisModuleKeyspaceSubscriber *sub = ln->value; + /* Only notify subscribers on events matching they registration, + * and avoid subscribers triggering themselves */ + if ((sub->event_mask & type) && sub->active == 0) { + RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.module = sub->module; + ctx.client = moduleKeyspaceSubscribersClient; + selectDb(ctx.client, dbid); + + /* mark the handler as activer to avoid reentrant loops. + * If the subscriber performs an action triggering itself, + * it will not be notified about it. */ + sub->active = 1; + sub->notify_callback(&ctx, type, event, key); + sub->active = 0; + moduleFreeContext(&ctx); + } + } +} + +/* Unsubscribe any notification subscirbers this module has upon unloading */ +void moduleUnsubscribeNotifications(RedisModule *module) { + listIter li; + listNode *ln; + listRewind(moduleKeyspaceSubscribers,&li); + while((ln = listNext(&li))) { + RedisModuleKeyspaceSubscriber *sub = ln->value; + if (sub->module == module) { + listDelNode(moduleKeyspaceSubscribers, ln); + zfree(sub); + } + } +} + /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ @@ -3706,9 +3845,14 @@ void moduleInitModulesSystem(void) { moduleUnblockedClients = listCreate(); - server.loadmodule_queue = listCreate(); modules = dictCreate(&modulesDictType,NULL); + + /* Set up the keyspace notification susbscriber list and static client */ + moduleKeyspaceSubscribers = listCreate(); + moduleKeyspaceSubscribersClient = createClient(-1); + moduleKeyspaceSubscribersClient->flags |= CLIENT_MODULE; + moduleRegisterCoreAPI(); if (pipe(server.module_blocked_pipe) == -1) { serverLog(LL_WARNING, @@ -3819,6 +3963,7 @@ return C_OK; } + /* Unload the module registered with the specified name. On success * C_OK is returned, otherwise C_ERR is returned and errno is set * to the following values depending on the type of error: @@ -3840,6 +3985,9 @@ moduleUnregisterCommands(module); + /* Remvoe any noification subscribers this module might have */ + moduleUnsubscribeNotifications(module); + /* Unregister all the hooks. TODO: Yet no hooks support here. */ /* Unload the dynamic library. */ @@ -4037,4 +4185,5 @@ REGISTER_API(DigestAddStringBuffer); REGISTER_API(DigestAddLongLong); REGISTER_API(DigestEndSequence); + REGISTER_API(SubscribeToKeyspaceEvents); } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/modules/testmodule.c new/redis-4.0.9/src/modules/testmodule.c --- old/redis-4.0.8/src/modules/testmodule.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/modules/testmodule.c 2018-03-26 18:04:15.000000000 +0200 @@ -30,6 +30,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#define REDISMODULE_EXPERIMENTAL_API #include "../redismodule.h" #include <string.h> @@ -124,6 +125,7 @@ RedisModule_ReplyWithError(ctx, msg); return REDISMODULE_ERR; } + int TestUnlink(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { RedisModule_AutoMemory(ctx); REDISMODULE_NOT_USED(argv); @@ -153,80 +155,153 @@ } +int NotifyCallback(RedisModuleCtx *ctx, int type, const char *event, + RedisModuleString *key) { + /* Increment a counter on the notifications: for each key notified we + * increment a counter */ + RedisModule_Log(ctx, "notice", "Got event type %d, event %s, key %s", type, + event, RedisModule_StringPtrLen(key, NULL)); + + RedisModule_Call(ctx, "HINCRBY", "csc", "notifications", key, "1"); + return REDISMODULE_OK; +} + +/* TEST.NOTIFICATIONS -- Test Keyspace Notifications. */ +int TestNotifications(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + +#define FAIL(msg, ...) \ + { \ + RedisModule_Log(ctx, "warning", "Failed NOTIFY Test. Reason: " #msg, ##__VA_ARGS__); \ + goto err; \ + } + RedisModule_Call(ctx, "FLUSHDB", ""); + + RedisModule_Call(ctx, "SET", "cc", "foo", "bar"); + RedisModule_Call(ctx, "SET", "cc", "foo", "baz"); + RedisModule_Call(ctx, "SADD", "cc", "bar", "x"); + RedisModule_Call(ctx, "SADD", "cc", "bar", "y"); + + RedisModule_Call(ctx, "HSET", "ccc", "baz", "x", "y"); + /* LPUSH should be ignored and not increment any counters */ + RedisModule_Call(ctx, "LPUSH", "cc", "l", "y"); + RedisModule_Call(ctx, "LPUSH", "cc", "l", "y"); + + size_t sz; + const char *rep; + RedisModuleCallReply *r = RedisModule_Call(ctx, "HGET", "cc", "notifications", "foo"); + if (r == NULL || RedisModule_CallReplyType(r) != REDISMODULE_REPLY_STRING) { + FAIL("Wrong or no reply for foo"); + } else { + rep = RedisModule_CallReplyStringPtr(r, &sz); + if (sz != 1 || *rep != '2') { + FAIL("Got reply '%s'. expected '2'", RedisModule_CallReplyStringPtr(r, NULL)); + } + } + + r = RedisModule_Call(ctx, "HGET", "cc", "notifications", "bar"); + if (r == NULL || RedisModule_CallReplyType(r) != REDISMODULE_REPLY_STRING) { + FAIL("Wrong or no reply for bar"); + } else { + rep = RedisModule_CallReplyStringPtr(r, &sz); + if (sz != 1 || *rep != '2') { + FAIL("Got reply '%s'. expected '2'", rep); + } + } + + r = RedisModule_Call(ctx, "HGET", "cc", "notifications", "baz"); + if (r == NULL || RedisModule_CallReplyType(r) != REDISMODULE_REPLY_STRING) { + FAIL("Wrong or no reply for baz"); + } else { + rep = RedisModule_CallReplyStringPtr(r, &sz); + if (sz != 1 || *rep != '1') { + FAIL("Got reply '%.*s'. expected '1'", sz, rep); + } + } + /* For l we expect nothing since we didn't subscribe to list events */ + r = RedisModule_Call(ctx, "HGET", "cc", "notifications", "l"); + if (r == NULL || RedisModule_CallReplyType(r) != REDISMODULE_REPLY_NULL) { + FAIL("Wrong reply for l"); + } + + RedisModule_Call(ctx, "FLUSHDB", ""); + + return RedisModule_ReplyWithSimpleString(ctx, "OK"); +err: + RedisModule_Call(ctx, "FLUSHDB", ""); + + return RedisModule_ReplyWithSimpleString(ctx, "ERR"); +} + /* TEST.CTXFLAGS -- Test GetContextFlags. */ int TestCtxFlags(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { REDISMODULE_NOT_USED(argc); REDISMODULE_NOT_USED(argv); - + RedisModule_AutoMemory(ctx); - + int ok = 1; const char *errString = NULL; - - #define FAIL(msg) \ - { \ - ok = 0; \ - errString = msg; \ - goto end; \ +#undef FAIL +#define FAIL(msg) \ + { \ + ok = 0; \ + errString = msg; \ + goto end; \ } - + int flags = RedisModule_GetContextFlags(ctx); if (flags == 0) { - FAIL("Got no flags"); + FAIL("Got no flags"); } - + if (flags & REDISMODULE_CTX_FLAGS_LUA) FAIL("Lua flag was set"); if (flags & REDISMODULE_CTX_FLAGS_MULTI) FAIL("Multi flag was set"); - + if (flags & REDISMODULE_CTX_FLAGS_AOF) FAIL("AOF Flag was set") /* Enable AOF to test AOF flags */ RedisModule_Call(ctx, "config", "ccc", "set", "appendonly", "yes"); flags = RedisModule_GetContextFlags(ctx); - if (!(flags & REDISMODULE_CTX_FLAGS_AOF)) - FAIL("AOF Flag not set after config set"); - + if (!(flags & REDISMODULE_CTX_FLAGS_AOF)) FAIL("AOF Flag not set after config set"); + if (flags & REDISMODULE_CTX_FLAGS_RDB) FAIL("RDB Flag was set"); /* Enable RDB to test RDB flags */ RedisModule_Call(ctx, "config", "ccc", "set", "save", "900 1"); flags = RedisModule_GetContextFlags(ctx); - if (!(flags & REDISMODULE_CTX_FLAGS_RDB)) - FAIL("RDB Flag was not set after config set"); - + if (!(flags & REDISMODULE_CTX_FLAGS_RDB)) FAIL("RDB Flag was not set after config set"); + if (!(flags & REDISMODULE_CTX_FLAGS_MASTER)) FAIL("Master flag was not set"); if (flags & REDISMODULE_CTX_FLAGS_SLAVE) FAIL("Slave flag was set"); if (flags & REDISMODULE_CTX_FLAGS_READONLY) FAIL("Read-only flag was set"); if (flags & REDISMODULE_CTX_FLAGS_CLUSTER) FAIL("Cluster flag was set"); - + if (flags & REDISMODULE_CTX_FLAGS_MAXMEMORY) FAIL("Maxmemory flag was set"); - ; + RedisModule_Call(ctx, "config", "ccc", "set", "maxmemory", "100000000"); flags = RedisModule_GetContextFlags(ctx); if (!(flags & REDISMODULE_CTX_FLAGS_MAXMEMORY)) - FAIL("Maxmemory flag was not set after config set"); - + FAIL("Maxmemory flag was not set after config set"); + if (flags & REDISMODULE_CTX_FLAGS_EVICT) FAIL("Eviction flag was set"); - RedisModule_Call(ctx, "config", "ccc", "set", "maxmemory-policy", - "allkeys-lru"); + RedisModule_Call(ctx, "config", "ccc", "set", "maxmemory-policy", "allkeys-lru"); flags = RedisModule_GetContextFlags(ctx); - if (!(flags & REDISMODULE_CTX_FLAGS_EVICT)) - FAIL("Eviction flag was not set after config set"); - - end: + if (!(flags & REDISMODULE_CTX_FLAGS_EVICT)) FAIL("Eviction flag was not set after config set"); + +end: /* Revert config changes */ RedisModule_Call(ctx, "config", "ccc", "set", "appendonly", "no"); RedisModule_Call(ctx, "config", "ccc", "set", "save", ""); RedisModule_Call(ctx, "config", "ccc", "set", "maxmemory", "0"); RedisModule_Call(ctx, "config", "ccc", "set", "maxmemory-policy", "noeviction"); - + if (!ok) { - RedisModule_Log(ctx, "warning", "Failed CTXFLAGS Test. Reason: %s", - errString); - return RedisModule_ReplyWithSimpleString(ctx, "ERR"); + RedisModule_Log(ctx, "warning", "Failed CTXFLAGS Test. Reason: %s", errString); + return RedisModule_ReplyWithSimpleString(ctx, "ERR"); } - - return RedisModule_ReplyWithSimpleString(ctx, "OK"); - } + return RedisModule_ReplyWithSimpleString(ctx, "OK"); +} /* ----------------------------- Test framework ----------------------------- */ @@ -310,6 +385,9 @@ T("test.string.printf", "cc", "foo", "bar"); if (!TestAssertStringReply(ctx,reply,"Got 3 args. argv[1]: foo, argv[2]: bar",38)) goto fail; + T("test.notify", ""); + if (!TestAssertStringReply(ctx,reply,"OK",2)) goto fail; + RedisModule_ReplyWithSimpleString(ctx,"ALL TESTS PASSED"); return REDISMODULE_OK; @@ -354,5 +432,14 @@ TestIt,"readonly",1,1,1) == REDISMODULE_ERR) return REDISMODULE_ERR; + RedisModule_SubscribeToKeyspaceEvents(ctx, + REDISMODULE_NOTIFY_HASH | + REDISMODULE_NOTIFY_SET | + REDISMODULE_NOTIFY_STRING, + NotifyCallback); + if (RedisModule_CreateCommand(ctx,"test.notify", + TestNotifications,"write deny-oom",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + return REDISMODULE_OK; } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/networking.c new/redis-4.0.9/src/networking.c --- old/redis-4.0.8/src/networking.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/networking.c 2018-03-26 18:04:15.000000000 +0200 @@ -376,6 +376,12 @@ addReplyString(c,"-ERR ",5); addReplyString(c,s,len); addReplyString(c,"\r\n",2); + if (c->flags & CLIENT_MASTER) { + char *cmdname = c->lastcmd ? c->lastcmd->name : "<unknown>"; + serverLog(LL_WARNING,"== CRITICAL == This slave is sending an error " + "to its master: '%s' after processing the command " + "'%s'", s, cmdname); + } } void addReplyError(client *c, const char *err) { @@ -1005,13 +1011,25 @@ /* Try to write buffers to the client socket. */ if (writeToClient(c->fd,c,0) == C_ERR) continue; - /* If there is nothing left, do nothing. Otherwise install - * the write handler. */ - if (clientHasPendingReplies(c) && - aeCreateFileEvent(server.el, c->fd, AE_WRITABLE, + /* If after the synchronous writes above we still have data to + * output to the client, we need to install the writable handler. */ + if (clientHasPendingReplies(c)) { + int ae_flags = AE_WRITABLE; + /* For the fsync=always policy, we want that a given FD is never + * served for reading and writing in the same event loop iteration, + * so that in the middle of receiving the query, and serving it + * to the client, we'll call beforeSleep() that will do the + * actual fsync of AOF to disk. AE_BARRIER ensures that. */ + if (server.aof_state == AOF_ON && + server.aof_fsync == AOF_FSYNC_ALWAYS) + { + ae_flags |= AE_BARRIER; + } + if (aeCreateFileEvent(server.el, c->fd, ae_flags, sendReplyToClient, c) == AE_ERR) - { - freeClientAsync(c); + { + freeClientAsync(c); + } } } return processed; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/notify.c new/redis-4.0.9/src/notify.c --- old/redis-4.0.8/src/notify.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/notify.c 2018-03-26 18:04:15.000000000 +0200 @@ -98,6 +98,12 @@ int len = -1; char buf[24]; + /* If any modules are interested in events, notify the module system now. + * This bypasses the notifications configuration, but the module engine + * will only call event subscribers if the event type matches the types + * they are interested in. */ + moduleNotifyKeyspaceEvent(type, event, key, dbid); + /* If notifications for this class of events are off, return ASAP. */ if (!(server.notify_keyspace_events & type)) return; @@ -115,7 +121,7 @@ decrRefCount(chanobj); } - /* __keyevente@<db>__:<event> <key> notifications. */ + /* __keyevent@<db>__:<event> <key> notifications. */ if (server.notify_keyspace_events & NOTIFY_KEYEVENT) { chan = sdsnewlen("__keyevent@",11); if (len == -1) len = ll2string(buf,sizeof(buf),dbid); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/object.c new/redis-4.0.9/src/object.c --- old/redis-4.0.8/src/object.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/object.c 2018-03-26 18:04:15.000000000 +0200 @@ -145,7 +145,7 @@ * * The 'humanfriendly' option is used for INCRBYFLOAT and HINCRBYFLOAT. */ robj *createStringObjectFromLongDouble(long double value, int humanfriendly) { - char buf[256]; + char buf[MAX_LONG_DOUBLE_CHARS]; int len = ld2string(buf,sizeof(buf),value,humanfriendly); return createStringObject(buf,len); } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/rdb.c new/redis-4.0.9/src/rdb.c --- old/redis-4.0.8/src/rdb.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/rdb.c 2018-03-26 18:04:15.000000000 +0200 @@ -830,9 +830,9 @@ ssize_t ret, len = 0; if ((ret = rdbSaveType(rdb,RDB_OPCODE_AUX)) == -1) return -1; len += ret; - if ((ret = rdbSaveRawString(rdb,key,keylen) == -1)) return -1; + if ((ret = rdbSaveRawString(rdb,key,keylen)) == -1) return -1; len += ret; - if ((ret = rdbSaveRawString(rdb,val,vallen) == -1)) return -1; + if ((ret = rdbSaveRawString(rdb,val,vallen)) == -1) return -1; len += ret; return len; } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/redis-cli.c new/redis-4.0.9/src/redis-cli.c --- old/redis-4.0.8/src/redis-cli.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/redis-cli.c 2018-03-26 18:04:15.000000000 +0200 @@ -2074,7 +2074,9 @@ #define TYPE_SET 2 #define TYPE_HASH 3 #define TYPE_ZSET 4 -#define TYPE_NONE 5 +#define TYPE_STREAM 5 +#define TYPE_NONE 6 +#define TYPE_COUNT 7 static redisReply *sendScan(unsigned long long *it) { redisReply *reply = redisCommand(context, "SCAN %llu", *it); @@ -2218,11 +2220,11 @@ } static void findBigKeys(void) { - unsigned long long biggest[5] = {0}, counts[5] = {0}, totalsize[5] = {0}; + unsigned long long biggest[TYPE_COUNT] = {0}, counts[TYPE_COUNT] = {0}, totalsize[TYPE_COUNT] = {0}; unsigned long long sampled = 0, total_keys, totlen=0, *sizes=NULL, it=0; - sds maxkeys[5] = {0}; - char *typename[] = {"string","list","set","hash","zset"}; - char *typeunit[] = {"bytes","items","members","fields","members"}; + sds maxkeys[TYPE_COUNT] = {0}; + char *typename[] = {"string","list","set","hash","zset","stream","none"}; + char *typeunit[] = {"bytes","items","members","fields","members","entries",""}; redisReply *reply, *keys; unsigned int arrsize=0, i; int type, *types=NULL; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/redismodule.h new/redis-4.0.9/src/redismodule.h --- old/redis-4.0.8/src/redismodule.h 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/redismodule.h 2018-03-26 18:04:15.000000000 +0200 @@ -82,6 +82,17 @@ #define REDISMODULE_CTX_FLAGS_EVICT 0x0200 +#define REDISMODULE_NOTIFY_GENERIC (1<<2) /* g */ +#define REDISMODULE_NOTIFY_STRING (1<<3) /* $ */ +#define REDISMODULE_NOTIFY_LIST (1<<4) /* l */ +#define REDISMODULE_NOTIFY_SET (1<<5) /* s */ +#define REDISMODULE_NOTIFY_HASH (1<<6) /* h */ +#define REDISMODULE_NOTIFY_ZSET (1<<7) /* z */ +#define REDISMODULE_NOTIFY_EXPIRED (1<<8) /* x */ +#define REDISMODULE_NOTIFY_EVICTED (1<<9) /* e */ +#define REDISMODULE_NOTIFY_ALL (REDISMODULE_NOTIFY_GENERIC | REDISMODULE_NOTIFY_STRING | REDISMODULE_NOTIFY_LIST | REDISMODULE_NOTIFY_SET | REDISMODULE_NOTIFY_HASH | REDISMODULE_NOTIFY_ZSET | REDISMODULE_NOTIFY_EXPIRED | REDISMODULE_NOTIFY_EVICTED) /* A */ + + /* A special pointer that we can use between the core and the module to signal * field deletion, and that is impossible to be a valid pointer. */ #define REDISMODULE_HASH_DELETE ((RedisModuleString*)(long)1) @@ -112,6 +123,7 @@ typedef int (*RedisModuleCmdFunc) (RedisModuleCtx *ctx, RedisModuleString **argv, int argc); +typedef int (*RedisModuleNotificationFunc) (RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key); typedef void *(*RedisModuleTypeLoadFunc)(RedisModuleIO *rdb, int encver); typedef void (*RedisModuleTypeSaveFunc)(RedisModuleIO *rdb, void *value); typedef void (*RedisModuleTypeRewriteFunc)(RedisModuleIO *aof, RedisModuleString *key, void *value); @@ -251,6 +263,8 @@ void REDISMODULE_API_FUNC(RedisModule_FreeThreadSafeContext)(RedisModuleCtx *ctx); void REDISMODULE_API_FUNC(RedisModule_ThreadSafeContextLock)(RedisModuleCtx *ctx); void REDISMODULE_API_FUNC(RedisModule_ThreadSafeContextUnlock)(RedisModuleCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_SubscribeToKeyspaceEvents)(RedisModuleCtx *ctx, int types, RedisModuleNotificationFunc cb); + #endif /* This is included inline inside each Redis module. */ @@ -372,6 +386,8 @@ REDISMODULE_GET_API(IsBlockedTimeoutRequest); REDISMODULE_GET_API(GetBlockedClientPrivateData); REDISMODULE_GET_API(AbortBlock); + REDISMODULE_GET_API(SubscribeToKeyspaceEvents); + #endif if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/server.c new/redis-4.0.9/src/server.c --- old/redis-4.0.8/src/server.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/server.c 2018-03-26 18:04:15.000000000 +0200 @@ -1433,6 +1433,7 @@ server.cluster_migration_barrier = CLUSTER_DEFAULT_MIGRATION_BARRIER; server.cluster_slave_validity_factor = CLUSTER_DEFAULT_SLAVE_VALIDITY; server.cluster_require_full_coverage = CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE; + server.cluster_slave_no_failover = CLUSTER_DEFAULT_SLAVE_NO_FAILOVER; server.cluster_configfile = zstrdup(CONFIG_DEFAULT_CLUSTER_CONFIG_FILE); server.cluster_announce_ip = CONFIG_DEFAULT_CLUSTER_ANNOUNCE_IP; server.cluster_announce_port = CONFIG_DEFAULT_CLUSTER_ANNOUNCE_PORT; @@ -1781,6 +1782,8 @@ server.stat_numcommands = 0; server.stat_numconnections = 0; server.stat_expiredkeys = 0; + server.stat_expired_stale_perc = 0; + server.stat_expired_time_cap_reached_count = 0; server.stat_evictedkeys = 0; server.stat_keyspace_misses = 0; server.stat_keyspace_hits = 0; @@ -3105,6 +3108,8 @@ "sync_partial_ok:%lld\r\n" "sync_partial_err:%lld\r\n" "expired_keys:%lld\r\n" + "expired_stale_perc:%.2f\r\n" + "expired_time_cap_reached_count:%lld\r\n" "evicted_keys:%lld\r\n" "keyspace_hits:%lld\r\n" "keyspace_misses:%lld\r\n" @@ -3129,6 +3134,8 @@ server.stat_sync_partial_ok, server.stat_sync_partial_err, server.stat_expiredkeys, + server.stat_expired_stale_perc*100, + server.stat_expired_time_cap_reached_count, server.stat_evictedkeys, server.stat_keyspace_hits, server.stat_keyspace_misses, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/server.h new/redis-4.0.9/src/server.h --- old/redis-4.0.8/src/server.h 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/server.h 2018-03-26 18:04:15.000000000 +0200 @@ -425,7 +425,7 @@ #define NOTIFY_ZSET (1<<7) /* z */ #define NOTIFY_EXPIRED (1<<8) /* x */ #define NOTIFY_EVICTED (1<<9) /* e */ -#define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED) /* A */ +#define NOTIFY_ALL (NOTIFY_GENERIC | NOTIFY_STRING | NOTIFY_LIST | NOTIFY_SET | NOTIFY_HASH | NOTIFY_ZSET | NOTIFY_EXPIRED | NOTIFY_EVICTED) /* A flag */ /* Get the first bind addr or NULL */ #define NET_FIRST_BIND_ADDR (server.bindaddr_count ? server.bindaddr[0] : NULL) @@ -938,6 +938,8 @@ long long stat_numcommands; /* Number of processed commands */ long long stat_numconnections; /* Number of connections received */ long long stat_expiredkeys; /* Number of expired keys */ + double stat_expired_stale_perc; /* Percentage of keys probably expired */ + long long stat_expired_time_cap_reached_count; /* Early expire cylce stops.*/ long long stat_evictedkeys; /* Number of evicted keys (maxmemory) */ long long stat_keyspace_hits; /* Number of successful lookups of keys */ long long stat_keyspace_misses; /* Number of failed lookups of keys */ @@ -1159,6 +1161,8 @@ int cluster_slave_validity_factor; /* Slave max data age for failover. */ int cluster_require_full_coverage; /* If true, put the cluster down if there is at least an uncovered slot.*/ + int cluster_slave_no_failover; /* Prevent slave from starting a failover + if the master is in failure state. */ char *cluster_announce_ip; /* IP address to announce on cluster bus. */ int cluster_announce_port; /* base port to announce on cluster bus. */ int cluster_announce_bus_port; /* bus port to announce on cluster bus. */ @@ -1321,6 +1325,8 @@ size_t moduleCount(void); void moduleAcquireGIL(void); void moduleReleaseGIL(void); +void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); + /* Utils */ long long ustime(void); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/t_hash.c new/redis-4.0.9/src/t_hash.c --- old/redis-4.0.8/src/t_hash.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/t_hash.c 2018-03-26 18:04:15.000000000 +0200 @@ -616,7 +616,7 @@ value += incr; - char buf[256]; + char buf[MAX_LONG_DOUBLE_CHARS]; int len = ld2string(buf,sizeof(buf),value,1); new = sdsnewlen(buf,len); hashTypeSet(o,c->argv[2]->ptr,new,HASH_SET_TAKE_VALUE); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/util.h new/redis-4.0.9/src/util.h --- old/redis-4.0.8/src/util.h 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/util.h 2018-03-26 18:04:15.000000000 +0200 @@ -33,6 +33,11 @@ #include <stdint.h> #include "sds.h" +/* The maximum number of characters needed to represent a long double + * as a string (long double has a huge range). + * This should be the size of the buffer given to ld2string */ +#define MAX_LONG_DOUBLE_CHARS 5*1024 + int stringmatchlen(const char *p, int plen, const char *s, int slen, int nocase); int stringmatch(const char *p, const char *s, int nocase); long long memtoll(const char *p, int *err); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/version.h new/redis-4.0.9/src/version.h --- old/redis-4.0.8/src/version.h 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/version.h 2018-03-26 18:04:15.000000000 +0200 @@ -1 +1 @@ -#define REDIS_VERSION "4.0.8" +#define REDIS_VERSION "4.0.9" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/src/ziplist.c new/redis-4.0.9/src/ziplist.c --- old/redis-4.0.8/src/ziplist.c 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/src/ziplist.c 2018-03-26 18:04:15.000000000 +0200 @@ -53,20 +53,20 @@ * <prevlen> <encoding> * * The length of the previous entry, <prevlen>, is encoded in the following way: - * If this length is smaller than 255 bytes, it will only consume a single + * If this length is smaller than 254 bytes, it will only consume a single * byte representing the length as an unsinged 8 bit integer. When the length - * is greater than or equal to 255, it will consume 5 bytes. The first byte is - * set to 255 (FF) to indicate a larger value is following. The remaining 4 + * is greater than or equal to 254, it will consume 5 bytes. The first byte is + * set to 254 (FE) to indicate a larger value is following. The remaining 4 * bytes take the length of the previous entry as value. * * So practically an entry is encoded in the following way: * - * <prevlen from 0 to 254> <encoding> <entry> + * <prevlen from 0 to 253> <encoding> <entry> * - * Or alternatively if the previous entry length is greater than 254 bytes + * Or alternatively if the previous entry length is greater than 253 bytes * the following encoding is used: * - * 0xFF <4 bytes unsigned little endian prevlen> <encoding> <entry> + * 0xFE <4 bytes unsigned little endian prevlen> <encoding> <entry> * * The encoding field of the entry depends on the content of the * entry. When the entry is a string, the first 2 bits of the encoding first diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/tests/cluster/cluster.tcl new/redis-4.0.9/tests/cluster/cluster.tcl --- old/redis-4.0.8/tests/cluster/cluster.tcl 2018-02-02 17:39:14.000000000 +0100 +++ new/redis-4.0.9/tests/cluster/cluster.tcl 2018-03-26 18:04:15.000000000 +0200 @@ -42,6 +42,16 @@ return {} } +# Get a specific node by ID by parsing the CLUSTER NODES output +# of the instance Number 'instance_id' +proc get_node_by_id {instance_id node_id} { + set nodes [get_cluster_nodes $instance_id] + foreach n $nodes { + if {[dict get $n id] eq $node_id} {return $n} + } + return {} +} + # Return the value of the specified CLUSTER INFO field. proc CI {n field} { get_info_field [R $n cluster info] $field diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/redis-4.0.8/tests/cluster/tests/13-no-failover-option.tcl new/redis-4.0.9/tests/cluster/tests/13-no-failover-option.tcl --- old/redis-4.0.8/tests/cluster/tests/13-no-failover-option.tcl 1970-01-01 01:00:00.000000000 +0100 +++ new/redis-4.0.9/tests/cluster/tests/13-no-failover-option.tcl 2018-03-26 18:04:15.000000000 +0200 @@ -0,0 +1,61 @@ +# Check that the no-failover option works + +source "../tests/includes/init-tests.tcl" + +test "Create a 5 nodes cluster" { + create_cluster 5 5 +} + +test "Cluster is up" { + assert_cluster_state ok +} + +test "Cluster is writable" { + cluster_write_test 0 +} + +test "Instance #5 is a slave" { + assert {[RI 5 role] eq {slave}} + + # Configure it to never failover the master + R 5 CONFIG SET cluster-slave-no-failover yes +} + +test "Instance #5 synced with the master" { + wait_for_condition 1000 50 { + [RI 5 master_link_status] eq {up} + } else { + fail "Instance #5 master link status is not up" + } +} + +test "The nofailover flag is propagated" { + set slave5_id [dict get [get_myself 5] id] + + foreach_redis_id id { + wait_for_condition 1000 50 { + [has_flag [get_node_by_id $id $slave5_id] nofailover] + } else { + fail "Instance $id can't see the nofailover flag of slave" + } + } +} + +set current_epoch [CI 1 cluster_current_epoch] + +test "Killing one master node" { + kill_instance redis 0 +} + +test "Cluster should be still down after some time" { + after 10000 + assert_cluster_state fail +} + +test "Instance #5 is still a slave" { + assert {[RI 5 role] eq {slave}} +} + +test "Restarting the previously killed master node" { + restart_instance redis 0 +}
