Hi,
I am trying to use the DLM userland API (libdlm3), and while I was able to
do plain lock acquisitions and conversions, I am stuck trying to update
and then read the lock value block.
Does anyone have working examples of this? I did look at the rhdlmbook
doc, but couldn't fine one.
Attached is a messy test I wrote, which fails because it looks like
up-converting a lock with the LKF_VALBLK set doesn't seem to overwrite the
buffer I provide for the lock value block (and with strace it looks like
the kernel device returns the LVB on a down-conversion! weird). Example
output below.
Cheers,
Jean-Marc
--
saff...@gmail.com
$ make D=1
gcc -D_REENTRANT -Wall -Werror -O0 -g locklvb.c -pthread -ldlm
-lpthread -o locklvb
$ ./locklvb
dlm_kernel_version 6.0.1
create_lockspace
create_lockspace: Operation not permitted
open_lockspace
dlm_pthread_init
acquiring NL on MyLock...
LOCK mode -> NL convert 0
read_lvb 0 write_lvb 0
completion ast
entering loop on lock #1
count 0
LOCK mode -> PW convert 1
read_lvb 0 write_lvb 0
completion ast
init lvb => 51
lvb cache => 52
LOCK mode -> CR convert 1
read_lvb 0 write_lvb 1
completion ast
count 1
LOCK mode -> PW convert 1
read_lvb 1 write_lvb 0
completion ast
read lvb -1
locklvb: locklvb.c:177: do_lock: Assertion `lvb_lock.val >= 0' failed.
Aborted (core dumped)
/* compile with:
gcc -o locklvb locklvb.c -pthread -ldlm
*/
#include <stdio.h>
#include <stdint.h>
#include <stdbool.h>
#include <stdlib.h>
#include <unistd.h>
#include <time.h>
#include <string.h>
#include <pthread.h>
#include <errno.h>
#include <assert.h>
#include <libdlm.h>
#define LOG(fmt, ...) \
do { \
printf(fmt "\n", ## __VA_ARGS__); \
} while (0)
#define DEBUG LOG
#if 1
#define MODE_LO LKM_CRMODE
#define MODE_HI LKM_PWMODE
#else
#define MODE_LO LKM_NLMODE
#define MODE_HI LKM_EXMODE
#endif
#define RESOURCE "MyLock"
#define LOCKSPACE "default"
/* state for completion ast */
static const char *resource = RESOURCE;
static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
static int done = 0;
static int seq = 0; /* sanity check */
static dlm_lshandle_t ls;
struct lvb_desc {
int valid;
union {
char buf[DLM_LVB_LEN]; /* 32 bytes */
long val;
};
};
static struct lvb_desc lvb_cache = { .valid = false };
static const char const *mode_str(int mode) {
switch(mode) {
#define MAP(_m) case LKM_ ## _m ## MODE: return # _m
MAP(NL);
MAP(CR);
MAP(CW);
MAP(PR);
MAP(PW);
MAP(EX);
#undef MAP
default: return "??";
}
}
static void astcb(void *arg)
{
LOG("completion ast");
pthread_mutex_lock(&mutex);
assert((intptr_t)arg == seq); /* sanity check */
done = 1;
pthread_cond_signal(&cond);
pthread_mutex_unlock(&mutex);
}
static void ast_wait(void)
{
pthread_mutex_lock(&mutex);
while (done == 0)
pthread_cond_wait(&cond, &mutex);
done = 0;
pthread_mutex_unlock(&mutex);
}
static void
do_lock(int mode, int flags, int *lockid,
const struct lvb_desc *out_lvb, /* write new lvb from this */
struct lvb_desc *in_lvb) /* read lvb and save it there */
{
int convert = (flags & LKF_CONVERT) != 0;
struct dlm_lksb lksb = {};
struct lvb_desc lvb_lock = {};
static int written = 0;
DEBUG("LOCK mode -> %s convert %d", mode_str(mode), convert);
if (convert)
lksb.sb_lkid = *lockid;
const int read_lvb =
written &&
convert && (mode == MODE_HI); /* up-convert */
const int write_lvb =
convert && (mode == MODE_LO); /* down-convert */
assert(!(read_lvb && write_lvb));
DEBUG("read_lvb %d write_lvb %d", read_lvb, write_lvb);
lvb_lock.val = -1; /* poison */
if (read_lvb || write_lvb) {
flags |= LKF_VALBLK;
lksb.sb_lvbptr = (char*)&lvb_lock.val;
}
if (read_lvb)
assert(out_lvb == NULL);
if (write_lvb) {
assert(in_lvb == NULL);
if (out_lvb->valid) {
assert(out_lvb->val > 0);
lvb_lock.val = out_lvb->val;
} else {
DEBUG("lvb invalidate");
flags |= LKF_IVVALBLK;
}
}
void *ast_arg = (void*)(intptr_t)++seq;
int rc = dlm_ls_lock(ls,
mode, &lksb, flags,
convert ? NULL : resource,
convert ? 0 : strlen(resource),
0, /* parent */
astcb, ast_arg, /* ast, ast arg */
NULL, NULL); /* bast, range */
if (rc) {
perror("dlm_lock");
exit(1);
}
ast_wait();
errno = lksb.sb_status;
if (lksb.sb_status) {
perror("dlm_lock post wait");
exit(1);
}
if (lksb.sb_flags)
DEBUG("sb_flags %d", lksb.sb_flags);
if ((lksb.sb_flags & DLM_SBF_VALNOTVALID) != 0)
DEBUG("DLM_SBF_VALNOTVALID");
if (read_lvb || write_lvb)
assert(lksb.sb_lvbptr == (char*)&lvb_lock.val);
if (read_lvb) {
/* "An LVB is valid when the lock manager first
creates the lock resource, in response to the first
lock request, before any client can assign a value
to the LVB."
So we never write 0 to the LVB, and always ignore 0
in the LVB.
*/
/* after comp ast runs, and despite poisoning, we
* shouldn't see a negative here! */
DEBUG("read lvb %ld", lvb_lock.val);
assert(lvb_lock.val >= 0);
in_lvb->valid =
((lksb.sb_flags & DLM_SBF_VALNOTVALID) == 0)
&& lvb_lock.val > 0;
if (in_lvb->valid)
in_lvb->val = lvb_lock.val;
}
if (write_lvb)
written = 1;
*lockid = lksb.sb_lkid;
}
static void
do_unlock(int lockid)
{
struct dlm_lksb lksb = {};
int rc = dlm_ls_unlock(ls, lockid, 0, &lksb, NULL);
if (rc) {
perror("dlm_unlock");
exit(1);
}
ast_wait();
errno = lksb.sb_status;
if (lksb.sb_status != 0
&& lksb.sb_status != EUNLOCK) {
perror("dlm_unlock post wait");
exit(1);
}
}
static int do_init(void)
{
int rc;
uint32_t major, minor, patch;
rc = dlm_kernel_version(&major, &minor, &patch);
if (rc != 0) {
perror("dlm_kernel_version");
return 1;
}
LOG("dlm_kernel_version %d.%d.%d", major, minor, patch);
LOG("create_lockspace");
ls = dlm_create_lockspace(LOCKSPACE, 0777); /* requires CAP_SYSADMIN */
if (ls == NULL) {
perror("create_lockspace");
if (errno != EEXIST && errno != EPERM)
return 1;
LOG("open_lockspace");
ls = dlm_open_lockspace(LOCKSPACE);
if (ls == NULL) {
perror("open_lockspace");
return 1;
}
}
LOG("dlm_pthread_init");
rc = dlm_ls_pthread_init(ls);
if(rc != 0) {
perror("dlm_pthread_init");
return 1;
}
return 0;
}
int main(int argc, char **argv) {
long count = 0;
int rc;
int lockid;
rc = do_init();
if (rc != 0)
return 1;
LOG("acquiring NL on %s...", resource);
do_lock(LKM_NLMODE, LKF_EXPEDITE, &lockid, NULL, NULL);
LOG("entering loop on lock #%d", lockid);
while (count < 10) {
DEBUG("count %ld", count);
struct lvb_desc const old_lvb = lvb_cache;
/* up-convert and read lvb */
do_lock(MODE_HI, LKF_CONVERT, &lockid, NULL, &lvb_cache);
if (old_lvb.valid != lvb_cache.valid) {
LOG("lvb valid => %s", lvb_cache.valid ? "true" : "false");
if (lvb_cache.valid)
LOG("lvb = %ld", lvb_cache.val);
} else if (old_lvb.valid && lvb_cache.valid) {
if (old_lvb.val != lvb_cache.val)
LOG("lvb %ld -> %ld", old_lvb.val, lvb_cache.val);
}
if (!lvb_cache.valid) {
lvb_cache.valid = true;
lvb_cache.val = 51;
LOG("init lvb => %ld", lvb_cache.val);
}
lvb_cache.val++;
DEBUG("lvb cache => %ld", lvb_cache.val);
/* down-convert and write lvb */
do_lock(MODE_LO, LKF_CONVERT, &lockid, &lvb_cache, NULL);
count++;
}
LOG("unlock");
do_unlock(lockid);
return 0;
}
--
Linux-cluster mailing list
Linux-cluster@redhat.com
https://www.redhat.com/mailman/listinfo/linux-cluster
