Hello, aside from working on files by buffering I/O, the BIO library also does *formatted* I/O, like serializing strings or integers.
I found myself in a situation where my client and my service are communicating using variable-length messages (containing a sequence of variable-length strings, mostly), so I'd have to parse the messages "by hand" using some kind of serialization. Since BIO has this formatted I/O, I thought I could "offload" this de/serialization to the library, but it works only on files and writing the message to a file and reading it back again is suboptimal. As such, I hacked together a set of functions in BIO to work with already allocated buffers, such as when dealing with messages received from a message queue. The patch is attached with this mail. Right now the API is written to cover my needs, so it might be lacking in some parts. Thanks, A.V.
>From c1951bac149e49ccf12d57ae59c891c2cee92424 Mon Sep 17 00:00:00 2001 From: Alessio Vanni <[email protected]> Date: Sat, 18 Apr 2020 16:20:27 +0200 Subject: [PATCH] Add formatted I/O to buffers allocated in memory --- src/include/gnunet_bio_lib.h | 219 ++++++++++++++++++++++ src/util/bio.c | 350 +++++++++++++++++++++++++++++++++++ src/util/test_bio.c | 102 ++++++++++ 3 files changed, 671 insertions(+) diff --git a/src/include/gnunet_bio_lib.h b/src/include/gnunet_bio_lib.h index 2f715ec97..f1d9734d9 100644 --- a/src/include/gnunet_bio_lib.h +++ b/src/include/gnunet_bio_lib.h @@ -308,6 +308,225 @@ GNUNET_BIO_write_int32 (struct GNUNET_BIO_WriteHandle *h, int32_t i); int GNUNET_BIO_write_int64 (struct GNUNET_BIO_WriteHandle *h, int64_t i); +/** + * Handle for I/O on an existing in-memory buffer. + */ +struct GNUNET_BIO_MemoryHandle; + +/** + * Create a handle. + * + * @param data buffer to use for I/O + * Note that hereafter, data will be managed by the handle, + * i.e. it will be freed when the handle is destroyed + * @param len size in bytes of data + * @return the handle or NULL on error + */ +struct GNUNET_BIO_MemoryHandle * +GNUNET_BIO_create (void *data, size_t len); + +/** + * Destroy a handle to an in-memory buffer. + * + * @param h the handle + * @param emsg set to the error message + * @return #GNUNET_OK on success, #GNUNET_SYSERR otherwise + */ +int +GNUNET_BIO_destroy (struct GNUNET_BIO_MemoryHandle *h, char **emsg); + +/** + * Reset the handle's built-in cursor. Useful to read data after being + * written in or to overwrite existing data. + * + * @param h the handle + * @return #GNUNET_OK on success, #GNUNET_SYSERR otherwise + */ +int +GNUNET_BIO_reset_cursor (struct GNUNET_BIO_MemoryHandle *h); + +/** + * Read the contents of an in-memory buffer into a buffer. + * + * @param h the handle to an in-memory buffer + * @param what describes what is being read (for error message creation) + * @param result the buffer to write the result to + * @param len the number of bytes to read + * @return #GNUNET_OK on success, #GNUNET_SYSERR otherwise + */ +int +GNUNET_BIO_read_memory (struct GNUNET_BIO_MemoryHandle *h, + const char *what, + void *result, + size_t len); + +/** + * Read the contents of an in-memory buffer into a buffer. + * + * @param h handle to an in-memory buffer + * @param file name of the source file + * @param line line number in the source file + * @param result the buffer to write the result to + * @param len the number of bytes to read + * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure + */ +int +GNUNET_BIO_read_mem (struct GNUNET_BIO_MemoryHandle *h, + const char *file, int line, + void *result, size_t size); + +/** + * Read 0-terminated string from an in-memory buffer. + * + * @param h the handle to an in-memory buffer + * @param what describes what is being read (for error message creation) + * @param result the buffer to store a pointer to the (allocated) string to + * (note that *result could be set to NULL as well) + * @param max_length maximum allowed length for the string + * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure + */ +int +GNUNET_BIO_read_string_memory (struct GNUNET_BIO_MemoryHandle *h, + const char *what, + char **result, + size_t max_length); + +/** + * Read a float. + * + * @param h hande to in-memory buffer + * @param f address of float to read + */ +#define GNUNET_BIO_read_float_memory(h, f) \ + (GNUNET_BIO_read_mem (h, \ + __FILE__, \ + __LINE__, \ + f, \ + sizeof(float))) + +/** + * Read a double. + * + * @param h hande to in-memory buffer + * @param f address of double to read + */ +#define GNUNET_BIO_read_double_memory(h, f) \ + (GNUNET_BIO_read_memory (h, \ + __FILE__, \ + __LINE__, \ + f, \ + sizeof(double))) + +/** + * Read an (u)int32_t. + * + * @param h handle to in-memory buffer + * @param file name of the source file + * @param line line number in the source file + * @param i address of 32-bit integer to read + * @return #GNUNET_OK on success, #GNUNET_SYSERR on error + */ +int +GNUNET_BIO_read_int32_memory__ (struct GNUNET_BIO_MemoryHandle *h, + const char *file, + int line, + int32_t *i); + +/** + * Read an (u)int32_t. + * + * @param h hande to in-memory buffer + * @param i address of 32-bit integer to read + */ +#define GNUNET_BIO_read_int32_memory(h, i) \ + GNUNET_BIO_read_int32_memory__ (h, __FILE__, __LINE__, i) + +/** + * Read an (u)int64_t. + * + * @param h handle to in-memory buffer + * @param file name of the source file + * @param line line number in the source file + * @param i address of 64-bit integer to read + * @return #GNUNET_OK on success, #GNUNET_SYSERR on error + */ +int +GNUNET_BIO_read_int64_memory__ (struct GNUNET_BIO_MemoryHandle *h, + const char *file, + int line, + int64_t *i); + +/** + * Read an (u)int64_t. + * + * @param h hande to in-memory buffer + * @param i address of 64-bit integer to read + */ +#define GNUNET_BIO_read_int64_memory(h, i) \ + GNUNET_BIO_read_int64_memory__ (h, __FILE__, __LINE__, i) + +/** + * Write a buffer to an in-memory buffer. + * + * @param h handle to in-memory buffer + * @param buffer the data to write + * @param n number of bytes to write + * @return #GNUNET_OK on success, #GNUNET_SYSERR on error + */ +int +GNUNET_BIO_write_memory (struct GNUNET_BIO_MemoryHandle *h, + const void *buffer, + size_t n); + +/** + * Write a string to an in-memory buffer. + * + * @param h handle to in-memory buffer + * @param s string to write (can be NULL) + * @return #GNUNET_OK on success, #GNUNET_SYSERR on error + */ +int +GNUNET_BIO_write_string_memory (struct GNUNET_BIO_MemoryHandle *h, const char *s); + +/** + * Write a float. + * + * @param h hande to in-memory buffer + * @param f float to write (must be a variable) + */ +#define GNUNET_BIO_write_float_memory(h, f) \ + GNUNET_BIO_write_memory (h, &f, sizeof(float)) + + +/** + * Write a double. + * + * @param h hande to in-memory buffer + * @param f double to write (must be a variable) + */ +#define GNUNET_BIO_write_double_memory(h, f) \ + GNUNET_BIO_write_memory (h, &f, sizeof(double)) + +/** + * Write an (u)int32_t. + * + * @param h hande to open file + * @param i 32-bit integer to write + * @return #GNUNET_OK on success, #GNUNET_SYSERR on error + */ +int +GNUNET_BIO_write_int32_memory (struct GNUNET_BIO_MemoryHandle *h, int32_t i); + +/** + * Write an (u)int64_t. + * + * @param h hande to in-memory buffer + * @param i 64-bit integer to write + * @return #GNUNET_OK on success, #GNUNET_SYSERR on error + */ +int +GNUNET_BIO_write_int64_memory (struct GNUNET_BIO_MemoryHandle *h, int64_t i); + #if 0 /* keep Emacsens' auto-indent happy */ { diff --git a/src/util/bio.c b/src/util/bio.c index e05258f73..254a1435a 100644 --- a/src/util/bio.c +++ b/src/util/bio.c @@ -604,5 +604,355 @@ GNUNET_BIO_write_int64 (struct GNUNET_BIO_WriteHandle *h, int64_t i) return GNUNET_BIO_write (h, &big, sizeof(int64_t)); } +/** + * Handle for I/O on an existing in-memory buffer. + */ +struct GNUNET_BIO_MemoryHandle +{ + /** + * Error message, NULL if there were no errors. + */ + char *emsg; + + /** + * The buffer passed as argument to the create function. + */ + char *buffer; + + /** + * Total size of @e buffer. + */ + size_t size; + + /** + * Current read/write offset in @e buffer. + */ + size_t pos; +}; + +/** + * Create a handle. + * + * @param data buffer to use for I/O + * Please note that @a data will not be freed when the handle is + * destroyed + * @param len size in bytes of data + * @return the handle or NULL on error + */ +struct GNUNET_BIO_MemoryHandle * +GNUNET_BIO_create (void *data, size_t len) +{ + struct GNUNET_BIO_MemoryHandle *h; + + h = GNUNET_new (struct GNUNET_BIO_MemoryHandle); + h->buffer = data; + h->size = len; + return h; +} + +/** + * Destroy a handle to an in-memory buffer. + * + * @param h the handle + * @param emsg set to the error message + * @return #GNUNET_OK on success, #GNUNET_SYSERR otherwise + */ +int +GNUNET_BIO_destroy (struct GNUNET_BIO_MemoryHandle *h, char **emsg) +{ + int err; + + err = (NULL == h->emsg) ? GNUNET_OK : GNUNET_SYSERR; + if (NULL != emsg) + *emsg = h->emsg; + else + GNUNET_free_non_null (h->emsg); + GNUNET_free(h); + return err; +} + +/** + * Reset the handle's built-in cursor. Useful to read data after being + * written in or to overwrite existing data. + * + * @param h the handle + * @return #GNUNET_OK on success, #GNUNET_SYSERR otherwise + */ +int +GNUNET_BIO_reset_cursor (struct GNUNET_BIO_MemoryHandle *h) { + if (NULL != h->emsg) + return GNUNET_SYSERR; + + h->pos = 0; + return GNUNET_OK; +} + +/** + * Read the contents of an in-memory buffer into a buffer. + * + * @param h the handle to an in-memory buffer + * @param what describes what is being read (for error message creation) + * @param result the buffer to write the result to + * @param len the number of bytes to read + * @return #GNUNET_OK on success, #GNUNET_SYSERR otherwise + */ +int +GNUNET_BIO_read_memory (struct GNUNET_BIO_MemoryHandle *h, + const char *what, + void *result, + size_t len) +{ + char *dst = result; + size_t min; + + if (NULL != h->emsg) + return GNUNET_SYSERR; + + if (len > h->size) + { + GNUNET_asprintf (&h->emsg, + _ ("`%s' requires too much space (%u > %u"), + what, + len, + h->size); + return GNUNET_SYSERR; + } + + min = (h->size - h->pos < len) ? h->size - h->pos : len; + if (0 == min) + { + GNUNET_asprintf (&h->emsg, + _ ("End reading `%s': %s"), + what, + _ ("End of data")); + return GNUNET_SYSERR; + } + + memset (dst, '\0', len); + GNUNET_memcpy (dst, h->buffer+h->pos, min); + h->pos += min; + return GNUNET_OK; +} + +/** + * Read the contents of an in-memory buffer into a buffer. + * + * @param h handle to an in-memory buffer + * @param file name of the source file + * @param line line number in the source file + * @param result the buffer to write the result to + * @param len the number of bytes to read + * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure + */ +int +GNUNET_BIO_read_mem (struct GNUNET_BIO_MemoryHandle *h, + const char *file, int line, + void *result, size_t size) +{ + char what[PATH_MAX + 1024]; + + GNUNET_snprintf (what, sizeof(what), "%s:%d", file, line); + return GNUNET_BIO_read_memory (h, what, result, size); +} + +/** + * Read 0-terminated string from an in-memory buffer. + * + * @param h the handle to an in-memory buffer + * @param what describes what is being read (for error message creation) + * @param result the buffer to store a pointer to the (allocated) string to + * (note that *result could be set to NULL as well) + * @param max_length maximum allowed length for the string + * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure + */ +int +GNUNET_BIO_read_string_memory (struct GNUNET_BIO_MemoryHandle *h, + const char *what, + char **result, + size_t max_length) +{ + char *buf; + int32_t big; + + if (GNUNET_OK != GNUNET_BIO_read_int32_memory (h, &big)) + { + GNUNET_free_non_null (h->emsg); + GNUNET_asprintf (&h->emsg, _ ("Error reading length of string `%s'"), what); + return GNUNET_SYSERR; + } + + if (0 == big) + { + *result = NULL; + return GNUNET_OK; + } + + if (big > max_length) + { + GNUNET_asprintf (&h->emsg, + _ ("String `%s' longer than allowed (%u > %u)"), + what, + big, + max_length); + return GNUNET_SYSERR; + } + + buf = GNUNET_malloc (big); + *result = buf; + buf [--big] = '\0'; + + if (0 == big) + return GNUNET_OK; + + if (GNUNET_OK != GNUNET_BIO_read_memory (h, what, buf, big)) + { + GNUNET_free (buf); + *result = NULL; + return GNUNET_SYSERR; + } + + return GNUNET_OK; +} + +/** + * Read an (u)int32_t. + * + * @param h handle to in-memory buffer + * @param file name of the source file + * @param line line number in the source file + * @param i address of 32-bit integer to read + * @return #GNUNET_OK on success, #GNUNET_SYSERR on error + */ +int +GNUNET_BIO_read_int32_memory__ (struct GNUNET_BIO_MemoryHandle *h, + const char *file, + int line, + int32_t *i) +{ + int32_t big; + + if (GNUNET_OK != GNUNET_BIO_read_mem (h, file, line, &big, sizeof(int32_t))) + return GNUNET_SYSERR; + *i = ntohl (big); + return GNUNET_OK; +} + +/** + * Read an (u)int64_t. + * + * @param h handle to in-memory buffer + * @param file name of the source file + * @param line line number in the source file + * @param i address of 64-bit integer to read + * @return #GNUNET_OK on success, #GNUNET_SYSERR on error + */ +int +GNUNET_BIO_read_int64_memory__ (struct GNUNET_BIO_MemoryHandle *h, + const char *file, + int line, + int64_t *i) +{ + int64_t big; + + if (GNUNET_OK != GNUNET_BIO_read_mem (h, file, line, &big, sizeof(int64_t))) + return GNUNET_SYSERR; + *i = GNUNET_ntohll (big); + return GNUNET_OK; +} + +/** + * Write a buffer to an in-memory buffer. + * + * @param h handle to in-memory buffer + * @param buffer the data to write + * @param n number of bytes to write + * @return #GNUNET_OK on success, #GNUNET_SYSERR on error + */ +int +GNUNET_BIO_write_memory (struct GNUNET_BIO_MemoryHandle *h, + const void *buffer, + size_t n) +{ + const char *src = buffer; + + if (NULL == h->buffer) + { + GNUNET_free_non_null (h->emsg); + GNUNET_asprintf (&h->emsg, _ ("No buffer to write to")); + return GNUNET_SYSERR; + } + + if (n > h->size || n > h->size - h->pos) + { + GNUNET_free_non_null (h->emsg); + GNUNET_asprintf (&h->emsg, _ ("No enough space in buffer")); + return GNUNET_SYSERR; + } + + GNUNET_memcpy (h->buffer+h->pos, src, n); + h->pos += n; + + return GNUNET_OK; +} + +/** + * Write a string to an in-memory buffer. + * + * @param h handle to in-memory buffer + * @param s string to write (can be NULL) + * @return #GNUNET_OK on success, #GNUNET_SYSERR on error + */ +int +GNUNET_BIO_write_string_memory (struct GNUNET_BIO_MemoryHandle *h, const char *s) +{ + uint32_t slen; + + slen = (uint32_t) ((NULL == s) ? 0 : strlen (s) + 1); + if (slen + sizeof(uint32_t) > h->size + || slen + sizeof(uint32_t) > h->size - h->pos) + { + GNUNET_free_non_null (h->emsg); + GNUNET_asprintf (&h->emsg, _ ("`%s' is too big for buffer"), s); + return GNUNET_SYSERR; + } + if (GNUNET_OK != GNUNET_BIO_write_int32_memory (h, slen)) + return GNUNET_SYSERR; + if (0 != slen) + return GNUNET_BIO_write_memory (h, s, slen - 1); + return GNUNET_OK; +} + +/** + * Write an (u)int32_t. + * + * @param h hande to open file + * @param i 32-bit integer to write + * @return #GNUNET_OK on success, #GNUNET_SYSERR on error + */ +int +GNUNET_BIO_write_int32_memory (struct GNUNET_BIO_MemoryHandle *h, int32_t i) +{ + int32_t big; + + big = htonl (i); + return GNUNET_BIO_write_memory (h, &big, sizeof(int32_t)); +} + +/** + * Write an (u)int64_t. + * + * @param h hande to in-memory buffer + * @param i 64-bit integer to write + * @return #GNUNET_OK on success, #GNUNET_SYSERR on error + */ +int +GNUNET_BIO_write_int64_memory (struct GNUNET_BIO_MemoryHandle *h, int64_t i) +{ + int64_t big; + + big = GNUNET_htonll (i); + return GNUNET_BIO_write_memory (h, &big, sizeof(int64_t)); +} + /* end of bio.c */ diff --git a/src/util/test_bio.c b/src/util/test_bio.c index 53b45c23a..a2d9a7844 100644 --- a/src/util/test_bio.c +++ b/src/util/test_bio.c @@ -384,6 +384,98 @@ test_fakebigmeta_rw () return 0; } +static int +test_normal_memory_rw (void) +{ + char *data = GNUNET_malloc(1024); + struct GNUNET_BIO_MemoryHandle *h; + char *str = NULL; + int32_t i = 0; + + h = GNUNET_BIO_create(data, 1024); + GNUNET_assert (NULL != h); + + GNUNET_assert (GNUNET_OK == GNUNET_BIO_write_string_memory (h, "test")); + GNUNET_assert (GNUNET_OK == GNUNET_BIO_write_int32_memory (h, 32)); + GNUNET_assert (GNUNET_OK == GNUNET_BIO_reset_cursor (h)); + GNUNET_assert (GNUNET_OK == + GNUNET_BIO_read_string_memory (h, "Read test string", + &str, 10)); + GNUNET_assert (GNUNET_OK == GNUNET_BIO_read_int32_memory (h, &i)); + + GNUNET_assert (NULL != str); + GNUNET_assert (0 == strcmp (str, "test")); + GNUNET_assert (32 == i); + + GNUNET_assert (GNUNET_OK == GNUNET_BIO_destroy (h, NULL)); + + GNUNET_free (str); + GNUNET_free (data); + return 0; +} + +static int +test_nullstring_memory_rw (void) +{ + char *data = GNUNET_malloc (1024); + struct GNUNET_BIO_MemoryHandle *h; + char *str = NULL; + + h = GNUNET_BIO_create (data, 1024); + GNUNET_assert (NULL != h); + + GNUNET_assert (GNUNET_OK == GNUNET_BIO_write_string_memory (h, NULL)); + GNUNET_assert (GNUNET_OK == GNUNET_BIO_reset_cursor (h)); + GNUNET_assert (GNUNET_OK == + GNUNET_BIO_read_string_memory (h, "Read string error", + &str, 10)); + GNUNET_assert (NULL == str); + + GNUNET_assert (GNUNET_OK == GNUNET_BIO_destroy (h, NULL)); + + GNUNET_free (data); + return 0; +} + +static int +test_emptystring_memory_rw (void) +{ + char *data = GNUNET_malloc (1024); + struct GNUNET_BIO_MemoryHandle *h; + char *str = NULL; + + h = GNUNET_BIO_create (data, 1024); + GNUNET_assert (NULL != h); + + GNUNET_assert (GNUNET_OK == GNUNET_BIO_write_string_memory (h, "")); + GNUNET_assert (GNUNET_OK == GNUNET_BIO_reset_cursor (h)); + GNUNET_assert (GNUNET_OK == + GNUNET_BIO_read_string_memory (h, "Read string error", + &str, 10)); + + GNUNET_assert (GNUNET_OK == GNUNET_BIO_destroy (h, NULL)); + + GNUNET_free (str); + GNUNET_free (data); + return 0; +} + +static int +test_bigstring_memory_rw (void) +{ + char *data = GNUNET_malloc (5); + struct GNUNET_BIO_MemoryHandle *h; + + h = GNUNET_BIO_create (data, 5); + GNUNET_assert (NULL != h); + + GNUNET_assert (GNUNET_OK != GNUNET_BIO_write_string_memory (h, "123456")); + GNUNET_assert (GNUNET_OK != GNUNET_BIO_destroy (h, NULL)); + + GNUNET_free (data); + return 0; +} + static int check_string_rw () @@ -416,6 +508,15 @@ check_file_rw () return 0; } +static int +check_memory_rw () +{ + GNUNET_assert (0 == test_normal_memory_rw ()); + GNUNET_assert (0 == test_nullstring_memory_rw ()); + GNUNET_assert (0 == test_emptystring_memory_rw ()); + GNUNET_assert (0 == test_bigstring_memory_rw ()); + return 0; +} int main (int argc, char *argv[]) @@ -424,6 +525,7 @@ main (int argc, char *argv[]) GNUNET_assert (0 == check_file_rw ()); GNUNET_assert (0 == check_metadata_rw ()); GNUNET_assert (0 == check_string_rw ()); + GNUNET_assert (0 == check_memory_rw ()); return 0; } -- 2.24.1
