Jan Friesse wrote:
> Attached patch is support for mmaped shm and POSIX semaphores for whitetank.
>
> It's little improved version of older patches, also merged into the one
> patch.
>
> Regards,
> Honza
>
Sorry bad patch was included (without proper free of buffer) in previous
mail.
Correct patch included there.
Regards,
Honza
diff --git a/branches/whitetank/Makefile.inc b/branches/whitetank/Makefile.inc
index c94e820..5d3add9 100644
--- a/branches/whitetank/Makefile.inc
+++ b/branches/whitetank/Makefile.inc
@@ -64,6 +64,9 @@ endif
# OPENAIS_PROFILE
+# Use POSIX IPC (sem_post, mmaped shm) instead of SYS V IPC (semop, shm)
+OPENAIS_POSIX_IPC=1
+
# default CFLAGS, LDFLAGS
#
CFLAGS =
@@ -101,6 +104,9 @@ ifeq (${OPENAIS_COMPAT}, LINUX)
override CFLAGS += -DOPENAIS_LINUX
override LDFLAGS += -ldl -lpthread
override DYFLAGS += -rdynamic
+ifdef OPENAIS_POSIX_IPC
+ override CFLAGS += -DOPENAIS_POSIX_IPC
+endif
endif
ifeq (${OPENAIS_COMPAT}, BSD)
override CFLAGS += -DOPENAIS_BSD
diff --git a/branches/whitetank/exec/ipc.c b/branches/whitetank/exec/ipc.c
index 5337d25..70dbe09 100644
--- a/branches/whitetank/exec/ipc.c
+++ b/branches/whitetank/exec/ipc.c
@@ -89,6 +89,12 @@
#include "util.h"
+#ifdef OPENAIS_POSIX_IPC
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#endif
+
#ifdef OPENAIS_SOLARIS
#define MSG_NOSIGNAL 0
#endif
@@ -138,10 +144,12 @@ struct conn_info {
enum conn_state state;
int notify_flow_control_enabled;
int refcount;
+#ifndef OPENAIS_POSIX_IPC
key_t shmkey;
key_t semkey;
int shmid;
int semid;
+#endif
unsigned int pending_semops;
pthread_mutex_t mutex;
struct shared_memory *mem;
@@ -157,7 +165,9 @@ static int shared_mem_dispatch_bytes_left (struct conn_info
*conn_info);
static void outq_flush (struct conn_info *conn_info);
+#ifndef OPENAIS_POSIX_IPC
static int priv_change (struct conn_info *conn_info);
+#endif
static void ipc_disconnect (struct conn_info *conn_info);
@@ -246,9 +256,16 @@ static inline int conn_info_destroy (struct conn_info
*conn_info)
/*
* Destroy shared memory segment and semaphore
*/
+#ifndef OPENAIS_POSIX_IPC
shmdt (conn_info->mem);
res = shmctl (conn_info->shmid, IPC_RMID, NULL);
semctl (conn_info->semid, 0, IPC_RMID);
+#else
+ sem_destroy (&conn_info->mem->sem0);
+ sem_destroy (&conn_info->mem->sem1);
+ sem_destroy (&conn_info->mem->sem2);
+ munmap (conn_info->mem, sizeof (struct shared_memory));
+#endif
/*
* Free allocated data needed to retry exiting library IPC connection
@@ -299,7 +316,9 @@ record_proc_state(const char *file, const char *function,
int line, const char *
static void *pthread_ipc_consumer (void *conn)
{
struct conn_info *conn_info = (struct conn_info *)conn;
+#ifndef OPENAIS_POSIX_IPC
struct sembuf sop;
+#endif
int res;
mar_req_header_t *header;
struct res_overlay res_overlay;
@@ -309,14 +328,17 @@ static void *pthread_ipc_consumer (void *conn)
int flow_control = 0;
for (;;) {
+#ifndef OPENAIS_POSIX_IPC
sop.sem_num = 0;
sop.sem_op = -1;
sop.sem_flg = 0;
+#endif
retry_semop:
if (ipc_thread_active (conn_info) == 0) {
openais_conn_refcount_dec (conn_info);
pthread_exit (0);
}
+#ifndef OPENAIS_POSIX_IPC
res = semop (conn_info->semid, &sop, 1);
if ((res == -1) && (errno == EINTR || errno == EAGAIN)) {
goto retry_semop;
@@ -325,7 +347,12 @@ retry_semop:
openais_conn_refcount_dec (conn_info);
pthread_exit (0);
}
-
+#else
+ res = sem_wait (&conn_info->mem->sem0);
+ if (res == -1 && errno == EINTR) {
+ goto retry_semop;
+ }
+#endif
openais_conn_refcount_inc (conn_info);
header = (mar_req_header_t *)conn_info->mem->req_buffer;
@@ -541,6 +568,34 @@ retry_recv:
return (0);
}
+#ifdef OPENAIS_POSIX_IPC
+static int
+memory_map (const char *path, size_t bytes, void **buf)
+{
+ int fd;
+ void *addr;
+ int res;
+
+ fd = open (path, O_RDWR, 0600);
+
+ unlink (path);
+
+ if (fd == -1) {
+ return (-1);
+ }
+
+ addr = mmap (NULL, bytes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+
+ res = close (fd);
+ if (res) {
+ return (-1);
+ }
+
+ *buf = addr;
+ return (0);
+}
+#endif
+
static int poll_handler_connection (
poll_handle handle,
int fd,
@@ -585,17 +640,23 @@ static int poll_handler_connection (
pthread_mutex_init (&conn_info->mutex, NULL);
req_setup = (mar_req_setup_t *)conn_info->setup_msg;
+#ifndef OPENAIS_POSIX_IPC
conn_info->shmkey = req_setup->shmkey;
conn_info->semkey = req_setup->semkey;
+#endif
conn_info->service = req_setup->service;
conn_info->refcount = 0;
conn_info->notify_flow_control_enabled = 0;
conn_info->setup_bytes_read = 0;
+#ifndef OPENAIS_POSIX_IPC
conn_info->shmid = shmget (conn_info->shmkey,
sizeof (struct shared_memory), 0600);
conn_info->mem = shmat (conn_info->shmid, NULL, 0);
conn_info->semid = semget (conn_info->semkey, 3, 0600);
+#else
+ res = memory_map (req_setup->shm_file, sizeof (struct
shared_memory), (void *)&conn_info->mem);
+#endif
conn_info->pending_semops = 0;
/*
@@ -642,11 +703,13 @@ static int poll_handler_connection (
case MESSAGE_REQ_OUTQ_FLUSH:
outq_flush (conn_info);
break;
+#ifndef OPENAIS_POSIX_IPC
case MESSAGE_REQ_CHANGE_EUID:
if (priv_change (conn_info) == -1) {
ipc_disconnect (conn_info);
}
break;
+#endif
default:
res = 0;
break;
@@ -893,17 +956,34 @@ void openais_ipc_exit (void)
{
struct list_head *list;
struct conn_info *conn_info;
+#ifdef OPENAIS_POSIX_IPC
+ void *retval;
+#endif
for (list = conn_info_list_head.next; list != &conn_info_list_head;
list = list->next) {
conn_info = list_entry (list, struct conn_info, list);
+#ifndef OPENAIS_POSIX_IPC
shmdt (conn_info->mem);
shmctl (conn_info->shmid, IPC_RMID, NULL);
semctl (conn_info->semid, 0, IPC_RMID);
-
+
pthread_kill (conn_info->thread, SIGUSR1);
+#else
+ if (conn_info->state != CONN_STATE_THREAD_ACTIVE &&
conn_info->state != CONN_STATE_THREAD_REQUEST_EXIT)
+ continue;
+
+ ipc_disconnect (conn_info);
+
+ pthread_join (conn_info->thread, &retval);
+
+ sem_destroy (&conn_info->mem->sem0);
+ sem_destroy (&conn_info->mem->sem1);
+ sem_destroy (&conn_info->mem->sem2);
+ munmap (conn_info->mem, sizeof (struct shared_memory));
+#endif
}
}
@@ -920,10 +1000,13 @@ void *openais_conn_private_data_get (void *conn)
int openais_response_send (void *conn, void *msg, int mlen)
{
struct conn_info *conn_info = (struct conn_info *)conn;
+#ifndef OPENAIS_POSIX_IPC
struct sembuf sop;
+#endif
int res;
memcpy (conn_info->mem->res_buffer, msg, mlen);
+#ifndef OPENAIS_POSIX_IPC
sop.sem_num = 1;
sop.sem_op = 1;
sop.sem_flg = 0;
@@ -936,13 +1019,21 @@ retry_semop:
if ((res == -1) && (errno == EINVAL || errno == EIDRM)) {
return (0);
}
+#else
+ res = sem_post (&conn_info->mem->sem1);
+ if (res == -1) {
+ return (-1);
+ }
+#endif
return (0);
}
int openais_response_iov_send (void *conn, struct iovec *iov, int iov_len)
{
struct conn_info *conn_info = (struct conn_info *)conn;
+#ifndef OPENAIS_POSIX_IPC
struct sembuf sop;
+#endif
int res;
int write_idx = 0;
int i;
@@ -952,6 +1043,7 @@ int openais_response_iov_send (void *conn, struct iovec
*iov, int iov_len)
write_idx += iov[i].iov_len;
}
+#ifndef OPENAIS_POSIX_IPC
sop.sem_num = 1;
sop.sem_op = 1;
sop.sem_flg = 0;
@@ -964,6 +1056,12 @@ retry_semop:
if ((res == -1) && (errno == EINVAL || errno == EIDRM)) {
return (0);
}
+#else
+ res = sem_post (&conn_info->mem->sem1);
+ if (res == -1) {
+ return (-1);
+ }
+#endif
return (0);
}
@@ -1011,7 +1109,9 @@ int memcpy_dwrap (struct conn_info *conn_info, void *msg,
int len)
void msg_send (void *conn, struct iovec *iov, int iov_len, int locked)
{
struct conn_info *conn_info = (struct conn_info *)conn;
+#ifndef OPENAIS_POSIX_IPC
struct sembuf sop;
+#endif
int res;
int i;
char buf;
@@ -1036,6 +1136,7 @@ void msg_send (void *conn, struct iovec *iov, int
iov_len, int locked)
if (res == -1) {
ipc_disconnect (conn_info);
}
+#ifndef OPENAIS_POSIX_IPC
sop.sem_num = 2;
sop.sem_op = 1;
sop.sem_flg = 0;
@@ -1048,6 +1149,12 @@ retry_semop:
if ((res == -1) && (errno == EINVAL || errno == EIDRM)) {
return;
}
+#else
+ res = sem_post (&conn_info->mem->sem2);
+ if (res == -1) {
+ return ;
+ }
+#endif
}
static void outq_flush (struct conn_info *conn_info) {
@@ -1085,6 +1192,7 @@ static void outq_flush (struct conn_info *conn_info) {
pthread_mutex_unlock (&conn_info->mutex);
}
+#ifndef OPENAIS_POSIX_IPC
static int priv_change (struct conn_info *conn_info)
{
mar_req_priv_change req_priv_change;
@@ -1128,6 +1236,7 @@ retry_recv:
}
return (0);
}
+#endif
static void msg_send_or_queue (void *conn, struct iovec *iov, int iov_len)
{
diff --git a/branches/whitetank/include/ipc_gen.h
b/branches/whitetank/include/ipc_gen.h
index a6235ee..251de1f 100644
--- a/branches/whitetank/include/ipc_gen.h
+++ b/branches/whitetank/include/ipc_gen.h
@@ -37,6 +37,10 @@
#include "mar_gen.h"
+#ifdef OPENAIS_POSIX_IPC
+#include <semaphore.h>
+#endif
+
enum service_types {
EVS_SERVICE = 0,
CLM_SERVICE = 1,
@@ -68,6 +72,11 @@ struct shared_memory {
unsigned char dispatch_buffer[DISPATCH_SIZE];
unsigned int read;
unsigned int write;
+#ifdef OPENAIS_POSIX_IPC
+ sem_t sem0;
+ sem_t sem1;
+ sem_t sem2;
+#endif
};
enum res_init_types {
@@ -81,8 +90,12 @@ typedef struct {
typedef struct {
int service __attribute__((aligned(8)));
+#ifndef OPENAIS_POSIX_IPC
unsigned long long shmkey __attribute__((aligned(8)));
unsigned long long semkey __attribute__((aligned(8)));
+#else
+ char shm_file[128] __attribute__((aligned(8)));
+#endif
} mar_req_setup_t __attribute__((aligned(8)));
typedef struct {
diff --git a/branches/whitetank/lib/Makefile b/branches/whitetank/lib/Makefile
index d4ddea5..ada6b0c 100644
--- a/branches/whitetank/lib/Makefile
+++ b/branches/whitetank/lib/Makefile
@@ -83,13 +83,13 @@ libevs.so.2.0.0: util.o evs.o
$(CC) $(LDFLAGS) $(DARWIN_OPTS) util.o evs.o -o $@
libcpg.so.2.0.0: util.o cpg.o
- $(CC) $(DARWIN_OPTS) util.o cpg.o -o $@
+ $(CC) $(LDFLAGS) $(DARWIN_OPTS) util.o cpg.o -o $@
libcfg.so.2.0.0: util.o cfg.o
- $(CC) $(DARWIN_OPTS) util.o cfg.o -o $@
+ $(CC) $(LDFLAGS) $(DARWIN_OPTS) util.o cfg.o -o $@
libconfdb.so.2.0.0: util.o cfg.o
- $(CC) -bundle -bind_at_load util.o confdb.o -o $@
+ $(CC) $(LDFLAGS) -bundle -bind_at_load util.o confdb.o -o $@
else
@@ -118,13 +118,13 @@ libevs.so.2.0.0: util.o evs.o
$(CC) $(LDFLAGS) -shared
-Wl,-soname,libevs.so.2,-version-script=libevs.versions util.o evs.o -o $@
libcpg.so.2.0.0: util.o cpg.o
- $(CC) -shared -Wl,-soname,libcpg.so.2,-version-script=libcpg.versions
util.o cpg.o -o $@
+ $(CC) $(LDFLAGS) -shared
-Wl,-soname,libcpg.so.2,-version-script=libcpg.versions util.o cpg.o -o $@
libcfg.so.2.0.0: util.o cfg.o
- $(CC) -shared -Wl,-soname,libcfg.so.2,-version-script=libcfg.versions
util.o cfg.o -o $@
+ $(CC) $(LDFLAGS) -shared
-Wl,-soname,libcfg.so.2,-version-script=libcfg.versions util.o cfg.o -o $@
libconfdb.so.2.0.0: util.o confdb.o
- $(CC) -shared
-Wl,-soname,libconfdb.so.2,-version-script=libconfdb.versions util.o confdb.o
-o $@
+ $(CC) $(LDFLAGS) -shared
-Wl,-soname,libconfdb.so.2,-version-script=libconfdb.versions util.o confdb.o
-o $@
endif
diff --git a/branches/whitetank/lib/util.c b/branches/whitetank/lib/util.c
index f285eb5..3d86b1c 100644
--- a/branches/whitetank/lib/util.c
+++ b/branches/whitetank/lib/util.c
@@ -70,6 +70,10 @@
#define IPC_SEMWAIT_TIMEOUT 2
#endif
+#ifdef OPENAIS_POSIX_IPC
+#include <sys/mman.h>
+#endif
+
enum SA_HANDLE_STATE {
SA_HANDLE_STATE_EMPTY,
SA_HANDLE_STATE_PENDINGREMOVAL,
@@ -85,8 +89,10 @@ struct saHandle {
struct ipc_segment {
int fd;
+#ifndef OPENAIS_POSIX_IPC
int shmid;
int semid;
+#endif
int flow_control_state;
struct shared_memory *shared_memory;
uid_t euid;
@@ -243,6 +249,7 @@ error_exit:
return (0);
}
+#ifndef OPENAIS_POSIX_IPC
static int
priv_change_send (struct ipc_segment *ipc_segment)
{
@@ -274,7 +281,7 @@ priv_change_send (struct ipc_segment *ipc_segment)
ipc_segment->euid = req_priv_change.euid;
return (0);
}
-
+#endif
#if defined(_SEM_SEMUN_UNDEFINED)
union semun {
@@ -285,6 +292,90 @@ union semun {
};
#endif
+#ifdef OPENAIS_POSIX_IPC
+static int
+memory_map (char *path, const char *file, void **buf, size_t bytes)
+{
+ int fd;
+ void *addr;
+ void *buffer;
+ char * const paths[] = {"/dev/shm/", "/var/run/", NULL};
+ char * const *path_iter = NULL;
+ ssize_t written;
+ size_t to_write;
+ long int page_size;
+ size_t i;
+ int res;
+
+ for (path_iter = paths; *path_iter != NULL; path_iter++) {
+ sprintf (path, "%s%s", *path_iter, file);
+ fd = mkstemp (path);
+ if (fd == -1) {
+ continue ;
+ }
+
+ res = ftruncate (fd, bytes);
+ if (res == -1) {
+ goto error_close_unlink;
+ }
+
+ /*
+ * Prealocate file
+ */
+ page_size = sysconf(_SC_PAGESIZE);
+ buffer = malloc (page_size);
+ if (buffer == NULL) {
+ goto error_close_unlink;
+ }
+
+ memset (buffer, 0, page_size);
+
+ for (i = 0; i < bytes; ) {
+retry_write:
+ if (i + page_size < bytes) {
+ to_write = page_size;
+ } else {
+ to_write = bytes - i;
+ }
+
+ written = write (fd, buffer, to_write);
+ if (written == 1 && errno == EINTR) {
+ goto retry_write;
+ }
+
+ if (written != to_write) {
+ goto error_free_close_unlink;
+ }
+
+ i += written;
+ }
+
+ free (buffer);
+
+ break ;
+error_free_close_unlink:
+ free (buffer);
+error_close_unlink:
+ close (fd);
+ unlink (path);
+ }
+
+ if (*path_iter == NULL) {
+ return (-1);
+ }
+
+ addr = mmap (NULL, bytes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+
+ res = close (fd);
+ if (res) {
+ return (-1);
+ }
+
+ *buf = addr;
+ return (0);
+}
+#endif
+
SaAisErrorT
openais_service_connect (
enum service_types service,
@@ -294,12 +385,16 @@ openais_service_connect (
struct sockaddr_un address;
SaAisErrorT error;
struct ipc_segment *ipc_segment;
+#ifndef OPENAIS_POSIX_IPC
key_t shmkey = 0;
key_t semkey = 0;
+ union semun semun;
+#else
+ char shm_path[128];
+#endif
int res;
mar_req_setup_t req_setup;
mar_res_setup_t res_setup;
- union semun semun;
res_setup.error = SA_AIS_ERR_LIBRARY;
@@ -332,6 +427,7 @@ openais_service_connect (
}
bzero (ipc_segment, sizeof (struct ipc_segment));
+#ifndef OPENAIS_POSIX_IPC
/*
* Allocate a shared memory segment
*/
@@ -357,7 +453,7 @@ openais_service_connect (
if (ipc_segment->shared_memory == (void *)-1) {
goto error_exit;
}
-
+
semun.val = 0;
res = semctl (ipc_segment->semid, 0, SETVAL, semun);
if (res != 0) {
@@ -373,9 +469,36 @@ openais_service_connect (
if (res != 0) {
goto error_exit;
}
+#else
+ /*
+ * Attach to shared memory segment
+ */
+ res = memory_map (shm_path, "openais_shm-XXXXXX", (void
*)&ipc_segment->shared_memory,
+ sizeof (struct shared_memory));
+ if (res == -1) {
+ return (-1);
+ }
+ /*
+ * Initialize semaphores
+ */
+ if (sem_init (&ipc_segment->shared_memory->sem0, 1, 0) == -1) {
+ return (-1);
+ }
+ if (sem_init (&ipc_segment->shared_memory->sem1, 1, 0) == -1) {
+ return (-1);
+ }
+ if (sem_init (&ipc_segment->shared_memory->sem2, 1, 0) == -1) {
+ return (-1);
+ }
+#endif
+
+#ifndef OPENAIS_POSIX_IPC
req_setup.shmkey = shmkey;
req_setup.semkey = semkey;
+#else
+ strcpy (req_setup.shm_file, shm_path);
+#endif
req_setup.service = service;
error = openais_send (request_fd, &req_setup, sizeof (mar_req_setup_t));
@@ -403,10 +526,17 @@ openais_service_connect (
error_exit:
close (request_fd);
+#ifndef OPENAIS_POSIX_IPC
if (ipc_segment->shmid > 0)
shmctl (ipc_segment->shmid, IPC_RMID, NULL);
if (ipc_segment->semid > 0)
semctl (ipc_segment->semid, 0, IPC_RMID);
+#else
+ sem_destroy (&ipc_segment->shared_memory->sem0);
+ sem_destroy (&ipc_segment->shared_memory->sem1);
+ sem_destroy (&ipc_segment->shared_memory->sem2);
+ munmap (ipc_segment->shared_memory, sizeof (struct shared_memory));
+#endif
return (res_setup.error);
}
@@ -471,23 +601,22 @@ ipc_sem_wait (
struct ipc_segment *ipc_segment,
int sem_num)
{
+#ifndef OPENAIS_POSIX_IPC
struct sembuf sop;
-#if defined(OPENAIS_LINUX)
+#else
struct timespec timeout;
struct pollfd pfd;
+ sem_t *sem;
#endif
int res;
+#ifndef OPENAIS_POSIX_IPC
sop.sem_num = sem_num;
sop.sem_op = -1;
sop.sem_flg = 0;
retry_semop:
-#if defined(OPENAIS_LINUX)
- timeout.tv_sec = IPC_SEMWAIT_TIMEOUT;
- timeout.tv_nsec = 0;
-
- res = semtimedop (ipc_segment->semid, &sop, 1, &timeout);
+ res = semop (ipc_segment->semid, &sop, 1);
if (res == -1 && errno == EINTR) {
goto retry_semop;
} else
@@ -495,7 +624,28 @@ retry_semop:
priv_change_send (ipc_segment);
goto retry_semop;
} else
- if (res == -1 && errno == EAGAIN) {
+ if (res == -1) {
+ return (SA_AIS_ERR_LIBRARY);
+ }
+#else
+ switch (sem_num) {
+ case 0:
+ sem = &ipc_segment->shared_memory->sem0;
+ break;
+ case 1:
+ sem = &ipc_segment->shared_memory->sem1;
+ break;
+ case 2:
+ sem = &ipc_segment->shared_memory->sem2;
+ break;
+ }
+
+retry_semwait:
+ timeout.tv_sec = time (NULL) + IPC_SEMWAIT_TIMEOUT;
+ timeout.tv_nsec = 0;
+
+ res = sem_timedwait (sem, &timeout);
+ if (res == -1 && errno == ETIMEDOUT) {
pfd.fd = ipc_segment->fd;
pfd.events = 0;
@@ -511,19 +661,10 @@ retry_semop:
}
}
- goto retry_semop;
+ goto retry_semwait;
} else
- if (res == -1) {
- return (SA_AIS_ERR_LIBRARY);
- }
-#else
- res = semop (ipc_segment->semid, &sop, 1);
if (res == -1 && errno == EINTR) {
- goto retry_semop;
- } else
- if (res == -1 && errno == EACCES) {
- priv_change_send (ipc_segment);
- goto retry_semop;
+ goto retry_semwait;
} else
if (res == -1) {
return (SA_AIS_ERR_LIBRARY);
@@ -633,7 +774,9 @@ openais_msg_send (
int iov_len)
{
struct ipc_segment *ipc_segment = (struct ipc_segment *)ipc_context;
+#ifndef OPENAIS_POSIX_IPC
struct sembuf sop;
+#endif
int i;
int res;
int req_buffer_idx = 0;
@@ -647,6 +790,8 @@ openais_msg_send (
iov[i].iov_len);
req_buffer_idx += iov[i].iov_len;
}
+
+#ifndef OPENAIS_POSIX_IPC
/*
* Signal semaphore #0 indicting a new message from client
* to server request queue
@@ -667,6 +812,13 @@ retry_semop:
if (res == -1) {
return (SA_AIS_ERR_LIBRARY);
}
+#else
+ res = sem_post (&ipc_segment->shared_memory->sem0);
+ if (res == -1) {
+ return (-1);
+ }
+#endif
+
return (SA_AIS_OK);
}
_______________________________________________
Openais mailing list
[email protected]
https://lists.linux-foundation.org/mailman/listinfo/openais
