This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Tarantool -- an efficient key/value data store".
The branch box-namespace_limit has been updated
via f74b8f1877963c179dfd8014694406064abc733d (commit)
from eade0287f8c53c3d490172af0ec44409d68750f8 (commit)
Summary of changes:
mod/box/box.h | 28 ++++++++++++------------
mod/box/box.m | 39 +++++++++++++++++++-------------
mod/box/index.h | 18 +++++++++------
mod/box/index.m | 53 ++++++++++++++++++++------------------------
test/box/show.result | 1 +
test/box_big/limit.result | 17 ++++++++++++++
test/box_big/limit.test | 11 +++++++++
7 files changed, 101 insertions(+), 66 deletions(-)
commit f74b8f1877963c179dfd8014694406064abc733d
Author: Roman Tokarev <[email protected]>
Date: Fri Apr 15 17:44:03 2011 +0400
A post-push fix for Blueprint: 'Limit the total amount of memory
consumed by a namespace'.
Don't update namespace usage during prepare phase, but capture
all usage into a txn and update namespace usage at the end of
commit phase.
Rename tnt_namespace_update_usage to tnt_namespace_capture_usage.
Add additional test cases for this blueprint.
diff --git a/mod/box/box.h b/mod/box/box.h
index 01bcf47..df27012 100644
--- a/mod/box/box.h
+++ b/mod/box/box.h
@@ -94,6 +94,8 @@ struct box_txn {
bool in_recover;
bool write_to_wal;
+
+ ssize_t usage;
};
enum tuple_flags {
@@ -147,21 +149,19 @@ enum box_mode {
ENUM(messages, MESSAGES);
static void __attribute__((unused, always_inline))
-tnt_namespace_update_usage(struct namespace *namespace, ssize_t bytes)
+tnt_namespace_capture_usage(struct box_txn *txn, ssize_t bytes)
{
- say_debug("usage = %zu limit = %zu bytes = %zi", namespace->usage,
namespace->limit, bytes);
-
- if (bytes < 0)
- assert(namespace->usage >= -bytes);
- else
- assert(namespace->usage <= SIZE_MAX - bytes);
- namespace->usage += bytes;
-
- if (namespace->usage > namespace->limit)
- if (bytes > 0) {
- tnt_raise(tnt_BoxException, reason:"namespace limit is
exceeded"
-
errcode:ERR_CODE_NAMESPACE_LIMIT);
- }
+ say_debug("namespace[%i]->usage = %zu namespace[%i]->limit = %zu "
+ "txn->usage = %zi bytes = %zi",
+ txn->n, namespace[txn->n].usage, txn->n,
namespace[txn->n].limit,
+ txn->usage, bytes);
+
+ txn->usage += bytes;
+
+ if (txn->usage > 0 && bytes > 0 &&
+ (namespace[txn->n].usage + txn->usage) > namespace[txn->n].limit)
+ tnt_raise(tnt_BoxException, reason:"namespace limit is exceeded"
+ errcode:ERR_CODE_NAMESPACE_LIMIT);
}
struct box_txn *txn_alloc(u32 flags);
diff --git a/mod/box/box.m b/mod/box/box.m
index 124b0d3..d2178f8 100644
--- a/mod/box/box.m
+++ b/mod/box/box.m
@@ -232,7 +232,7 @@ tuple_free(struct box_tuple *tuple)
}
static void
-tuple_ref(struct box_txn *txn, struct box_tuple *tuple, int count)
+tuple_ref(struct box_tuple *tuple, int count)
{
assert(tuple->refs + count >= 0);
tuple->refs += count;
@@ -240,10 +240,8 @@ tuple_ref(struct box_txn *txn, struct box_tuple *tuple,
int count)
if (tuple->refs > 0)
tuple->flags &= ~NEW;
- if (tuple->refs == 0) {
- tnt_namespace_update_usage(&namespace[txn->n],
-SIZEOF_TUPLE(tuple));
+ if (tuple->refs == 0)
tuple_free(tuple);
- }
}
void
@@ -251,7 +249,7 @@ tuple_txn_ref(struct box_txn *txn, struct box_tuple *tuple)
{
say_debug("tuple_txn_ref(%p)", tuple);
tbuf_append(txn->ref_tuples, &tuple, sizeof(struct box_tuple *));
- tuple_ref(txn, tuple, +1);
+ tuple_ref(tuple, +1);
}
static void __attribute__((noinline))
@@ -267,14 +265,16 @@ prepare_replace(struct box_txn *txn, size_t cardinality,
struct tbuf *data)
txn->tuple = tuple_alloc(data->len);
tuple_txn_ref(txn, txn->tuple);
- tnt_namespace_update_usage(&namespace[txn->n],
SIZEOF_TUPLE(txn->tuple));
txn->tuple->cardinality = cardinality;
memcpy(txn->tuple->data, data->data, data->len);
txn->old_tuple = txn->index->find_by_tuple(txn->index, txn->tuple);
- if (txn->old_tuple != NULL)
+ if (txn->old_tuple != NULL) {
tuple_txn_ref(txn, txn->old_tuple);
+ tnt_namespace_capture_usage(txn, -SIZEOF_TUPLE(txn->old_tuple));
+ }
+ tnt_namespace_capture_usage(txn, SIZEOF_TUPLE(txn->tuple));
if (txn->flags & BOX_ADD && txn->old_tuple != NULL)
tnt_raise(tnt_BoxException, reason:"tuple found"
errcode:ERR_CODE_NODE_FOUND);
@@ -316,7 +316,7 @@ prepare_replace(struct box_txn *txn, size_t cardinality,
struct tbuf *data)
* txn_commit().
*/
foreach_index(txn->n, index)
- index->replace(index, NULL, txn->tuple);
+ index->replace(txn,index, NULL, txn->tuple);
}
if (!(txn->flags & BOX_QUIET)) {
@@ -334,14 +334,14 @@ commit_replace(struct box_txn *txn)
{
if (txn->old_tuple != NULL) {
foreach_index(txn->n, index)
- index->replace(index, txn->old_tuple, txn->tuple);
+ index->replace(txn, index, txn->old_tuple, txn->tuple);
- tuple_ref(txn, txn->old_tuple, -1);
+ tuple_ref(txn->old_tuple, -1);
}
if (txn->tuple != NULL) {
txn->tuple->flags &= ~GHOST;
- tuple_ref(txn, txn->tuple, +1);
+ tuple_ref(txn->tuple, +1);
}
}
@@ -352,7 +352,7 @@ rollback_replace(struct box_txn *txn)
if (txn->tuple && txn->tuple->flags & GHOST) {
foreach_index(txn->n, index)
- index->remove(index, txn->tuple);
+ index->remove(txn, index, txn->tuple);
}
}
@@ -504,6 +504,7 @@ prepare_update_fields(struct box_txn *txn, struct tbuf
*data)
goto out;
}
+ tnt_namespace_capture_usage(txn, -SIZEOF_TUPLE(txn->old_tuple));
lock_tuple(txn, txn->old_tuple);
@@ -568,7 +569,7 @@ prepare_update_fields(struct box_txn *txn, struct tbuf
*data)
bsize += fields[i]->len + varint32_sizeof(fields[i]->len);
txn->tuple = tuple_alloc(bsize);
tuple_txn_ref(txn, txn->tuple);
- tnt_namespace_update_usage(&namespace[txn->n],
SIZEOF_TUPLE(txn->tuple));
+ tnt_namespace_capture_usage(txn, SIZEOF_TUPLE(txn->tuple));
txn->tuple->cardinality = txn->old_tuple->cardinality;
uint8_t *p = txn->tuple->data;
@@ -723,8 +724,9 @@ commit_delete(struct box_txn *txn)
return;
foreach_index(txn->n, index)
- index->remove(index, txn->old_tuple);
- tuple_ref(txn, txn->old_tuple, -1);
+ index->remove(txn, index, txn->old_tuple);
+ tuple_ref(txn->old_tuple, -1);
+ tnt_namespace_capture_usage(txn, -SIZEOF_TUPLE(txn->old_tuple));
}
static bool
@@ -739,6 +741,7 @@ txn_alloc(u32 flags)
struct box_txn *txn = p0alloc(fiber->pool, sizeof(*txn));
txn->ref_tuples = tbuf_alloc(fiber->pool);
txn->flags = flags;
+ txn->usage = 0;
return txn;
}
@@ -795,7 +798,7 @@ txn_cleanup(struct box_txn *txn)
while (i-- > 0) {
say_debug("tuple_txn_unref(%p)", *tuple);
- tuple_ref(txn, *tuple++, -1);
+ tuple_ref(*tuple++, -1);
}
/* mark txn as clean */
@@ -831,6 +834,8 @@ txn_commit(struct box_txn *txn)
commit_delete(txn);
else
commit_replace(txn);
+
+ namespace[txn->n].usage += txn->usage;
}
if (txn->flags & BOX_QUIET)
@@ -1601,6 +1606,8 @@ mod_info(struct tbuf *out)
tbuf_printf(out, " namespace_%u:" CRLF, i);
tbuf_printf(out, " usage: %zu" CRLF, namespace[i].usage);
+ tbuf_printf(out, " usage%%: %.1f" CRLF,
+ (float)(100 * namespace[i].usage) /
namespace[i].limit);
}
}
diff --git a/mod/box/index.h b/mod/box/index.h
index 7a0a2d1..54bdbd5 100644
--- a/mod/box/index.h
+++ b/mod/box/index.h
@@ -59,20 +59,25 @@ struct tree_index_member {
#define SIZEOF_TREE_INDEX_MEMBER(index) \
(sizeof(struct tree_index_member) + sizeof(struct field) *
(index)->key_cardinality)
+#define SIZEOF_HASH_INDEX_MEMBER(index) \
+ (sizeof((index)->idx.hash->keys[0]) +
sizeof((index)->idx.hash->vals[0]))
+
#include <third_party/sptree.h>
SPTREE_DEF(str_t, realloc);
+struct box_txn;
+
struct index {
bool enabled;
bool unique;
- struct box_tuple *(*find) (struct index * index, void *key); /* only
for unique lookups */
- struct box_tuple *(*find_by_tuple) (struct index * index, struct
box_tuple * pattern);
- void (*remove) (struct index * index, struct box_tuple *);
- void (*replace) (struct index * index, struct box_tuple *, struct
box_tuple *);
- void (*iterator_init) (struct index *, struct tree_index_member *
pattern);
- struct box_tuple *(*iterator_next) (struct index *, struct
tree_index_member * pattern);
+ struct box_tuple *(*find) (struct index * index, void *key); /* only
for unique lookups */
+ struct box_tuple *(*find_by_tuple) (struct index *, struct box_tuple *
pattern);
+ void (*remove) (struct box_txn *, struct index *, struct box_tuple *);
+ void (*replace) (struct box_txn *, struct index *, struct box_tuple *,
struct box_tuple *);
+ void (*iterator_init) (struct index *, struct tree_index_member *);
+ struct box_tuple *(*iterator_next) (struct index *, struct
tree_index_member *);
union {
khash_t(lstr_ptr_map) * str_hash;
khash_t(int_ptr_map) * int_hash;
@@ -116,7 +121,6 @@ struct tree_index_member * alloc_search_pattern(struct
index *index, int key_car
void index_iterator_init_tree_str(struct index *self, struct tree_index_member
*pattern);
struct box_tuple * index_iterator_next_tree_str(struct index *self, struct
tree_index_member *pattern);
-struct box_txn;
void validate_indeces(struct box_txn *txn);
void build_indexes(void);
diff --git a/mod/box/index.m b/mod/box/index.m
index e6a2fd1..5f360aa 100644
--- a/mod/box/index.m
+++ b/mod/box/index.m
@@ -296,7 +296,7 @@ index_find_tree_by_tuple(struct index *self, struct
box_tuple *tuple)
}
static void
-index_remove_hash_num(struct index *self, struct box_tuple *tuple)
+index_remove_hash_num(struct box_txn *txn, struct index *self, struct
box_tuple *tuple)
{
void *key = tuple_field(tuple, self->key_field->fieldno);
unsigned int key_size = load_varint32(&key);
@@ -306,14 +306,14 @@ index_remove_hash_num(struct index *self, struct
box_tuple *tuple)
tnt_raise(tnt_BoxException,
reason:"key is not u32"
errcode:ERR_CODE_ILLEGAL_PARAMS);
assoc_delete(int_ptr_map, self->idx.int_hash, num);
- tnt_namespace_update_usage(self->namespace, -(sizeof(num) +
sizeof(tuple)));
+ tnt_namespace_capture_usage(txn, -SIZEOF_HASH_INDEX_MEMBER(self));
#ifdef DEBUG
say_debug("index_remove_hash_num(self:%p, key:%i)", self, num);
#endif
}
static void
-index_remove_hash_num64(struct index *self, struct box_tuple *tuple)
+index_remove_hash_num64(struct box_txn *txn, struct index *self, struct
box_tuple *tuple)
{
void *key = tuple_field(tuple, self->key_field->fieldno);
unsigned int key_size = load_varint32(&key);
@@ -323,18 +323,18 @@ index_remove_hash_num64(struct index *self, struct
box_tuple *tuple)
tnt_raise(tnt_BoxException,
reason:"key is not u64"
errcode:ERR_CODE_ILLEGAL_PARAMS);
assoc_delete(int64_ptr_map, self->idx.int64_hash, num);
- tnt_namespace_update_usage(self->namespace, -(sizeof(num) +
sizeof(tuple)));
+ tnt_namespace_capture_usage(txn, -SIZEOF_HASH_INDEX_MEMBER(self));
#ifdef DEBUG
say_debug("index_remove_hash_num(self:%p, key:%"PRIu64")", self, num);
#endif
}
static void
-index_remove_hash_str(struct index *self, struct box_tuple *tuple)
+index_remove_hash_str(struct box_txn *txn, struct index *self, struct
box_tuple *tuple)
{
void *key = tuple_field(tuple, self->key_field->fieldno);
assoc_delete(lstr_ptr_map, self->idx.str_hash, key);
- tnt_namespace_update_usage(self->namespace, -(sizeof(key) +
sizeof(tuple)));
+ tnt_namespace_capture_usage(txn, -SIZEOF_HASH_INDEX_MEMBER(self));
#ifdef DEBUG
u32 size = load_varint32(&key);
say_debug("index_remove_hash_str(self:%p, key:'%.*s')", self, size, (u8
*)key);
@@ -342,23 +342,21 @@ index_remove_hash_str(struct index *self, struct
box_tuple *tuple)
}
static void
-index_remove_tree_str(struct index *self, struct box_tuple *tuple)
+index_remove_tree_str(struct box_txn *txn, struct index *self, struct
box_tuple *tuple)
{
struct tree_index_member *member = tuple2tree_index_member(self, tuple,
NULL);
sptree_str_t_delete(self->idx.tree, member);
- tnt_namespace_update_usage(self->namespace,
-SIZEOF_TREE_INDEX_MEMBER(self));
+ tnt_namespace_capture_usage(txn, -SIZEOF_TREE_INDEX_MEMBER(self));
}
static void
-index_replace_hash_num(struct index *self, struct box_tuple *old_tuple, struct
box_tuple *tuple)
+index_replace_hash_num(struct box_txn *txn, struct index *self,
+ struct box_tuple *old_tuple, struct box_tuple *tuple)
{
void *key = tuple_field(tuple, self->key_field->fieldno);
u32 key_size = load_varint32(&key);
u32 num = *(u32 *)key;
- /* see index_replace_tree_str for comments */
- tnt_namespace_update_usage(self->namespace, sizeof(num) +
sizeof(tuple));
-
if (key_size != 4)
tnt_raise(tnt_BoxException,
reason:"key is not u32"
errcode:ERR_CODE_ILLEGAL_PARAMS);
@@ -368,9 +366,11 @@ index_replace_hash_num(struct index *self, struct
box_tuple *old_tuple, struct b
load_varint32(&old_key);
u32 old_num = *(u32 *)old_key;
assoc_delete(int_ptr_map, self->idx.int_hash, old_num);
+ tnt_namespace_capture_usage(txn,
-SIZEOF_HASH_INDEX_MEMBER(self));
}
assoc_replace(int_ptr_map, self->idx.int_hash, num, tuple);
+ tnt_namespace_capture_usage(txn, SIZEOF_HASH_INDEX_MEMBER(self));
#ifdef DEBUG
say_debug("index_replace_hash_num(self:%p, old_tuple:%p, tuple:%p)
key:%i", self, old_tuple,
tuple, num);
@@ -378,15 +378,13 @@ index_replace_hash_num(struct index *self, struct
box_tuple *old_tuple, struct b
}
static void
-index_replace_hash_num64(struct index *self, struct box_tuple *old_tuple,
struct box_tuple *tuple)
+index_replace_hash_num64(struct box_txn *txn, struct index *self,
+ struct box_tuple *old_tuple, struct box_tuple *tuple)
{
void *key = tuple_field(tuple, self->key_field->fieldno);
u32 key_size = load_varint32(&key);
u64 num = *(u64 *)key;
- /* see index_replace_tree_str for comments */
- tnt_namespace_update_usage(self->namespace, sizeof(num) +
sizeof(tuple));
-
if (key_size != 8)
tnt_raise(tnt_BoxException,
reason:"key is not u64"
errcode:ERR_CODE_ILLEGAL_PARAMS);
@@ -396,9 +394,11 @@ index_replace_hash_num64(struct index *self, struct
box_tuple *old_tuple, struct
load_varint32(&old_key);
u64 old_num = *(u64 *)old_key;
assoc_delete(int64_ptr_map, self->idx.int64_hash, old_num);
+ tnt_namespace_capture_usage(txn,
-SIZEOF_HASH_INDEX_MEMBER(self));
}
assoc_replace(int64_ptr_map, self->idx.int64_hash, num, tuple);
+ tnt_namespace_capture_usage(txn, SIZEOF_HASH_INDEX_MEMBER(self));
#ifdef DEBUG
say_debug("index_replace_hash_num(self:%p, old_tuple:%p, tuple:%p)
key:%"PRIu64, self, old_tuple,
tuple, num);
@@ -406,13 +406,11 @@ index_replace_hash_num64(struct index *self, struct
box_tuple *old_tuple, struct
}
static void
-index_replace_hash_str(struct index *self, struct box_tuple *old_tuple, struct
box_tuple *tuple)
+index_replace_hash_str(struct box_txn *txn, struct index *self,
+ struct box_tuple *old_tuple, struct box_tuple *tuple)
{
void *key = tuple_field(tuple, self->key_field->fieldno);
- /* see index_replace_tree_str for comments */
- tnt_namespace_update_usage(self->namespace, sizeof(key) +
sizeof(tuple));
-
if (key == NULL)
tnt_raise(tnt_BoxException,
reason:"Supplied tuple misses a field which is part
of an index"
@@ -421,9 +419,11 @@ index_replace_hash_str(struct index *self, struct
box_tuple *old_tuple, struct b
if (old_tuple != NULL) {
void *old_key = tuple_field(old_tuple,
self->key_field->fieldno);
assoc_delete(lstr_ptr_map, self->idx.str_hash, old_key);
+ tnt_namespace_capture_usage(txn,
-SIZEOF_HASH_INDEX_MEMBER(self));
}
assoc_replace(lstr_ptr_map, self->idx.str_hash, key, tuple);
+ tnt_namespace_capture_usage(txn, SIZEOF_HASH_INDEX_MEMBER(self));
#ifdef DEBUG
u32 size = load_varint32(&key);
say_debug("index_replace_hash_str(self:%p, old_tuple:%p, tuple:%p)
key:'%.*s'", self,
@@ -432,15 +432,9 @@ index_replace_hash_str(struct index *self, struct
box_tuple *old_tuple, struct b
}
static void
-index_replace_tree_str(struct index *self, struct box_tuple *old_tuple, struct
box_tuple *tuple)
+index_replace_tree_str(struct box_txn *txn, struct index *self,
+ struct box_tuple *old_tuple, struct box_tuple *tuple)
{
- /*
- * Update usage at the beggining because if any error have been
- * occured there is no way to determine whether update of index was
- * successfull. So we always decrease usage during index cleanup.
- */
- tnt_namespace_update_usage(self->namespace,
SIZEOF_TREE_INDEX_MEMBER(self));
-
if (tuple->cardinality < self->field_cmp_order_cnt)
tnt_raise(tnt_BoxException,
reason:"Supplied tuple misses a field which is part
of an index"
@@ -449,8 +443,9 @@ index_replace_tree_str(struct index *self, struct box_tuple
*old_tuple, struct b
struct tree_index_member *member = tuple2tree_index_member(self, tuple,
NULL);
if (old_tuple)
- index_remove_tree_str(self, old_tuple);
+ index_remove_tree_str(txn, self, old_tuple);
sptree_str_t_insert(self->idx.tree, member);
+ tnt_namespace_capture_usage(txn, SIZEOF_TREE_INDEX_MEMBER(self));
}
void
diff --git a/test/box/show.result b/test/box/show.result
index 8860e40..647bcce 100644
--- a/test/box/show.result
+++ b/test/box/show.result
@@ -109,4 +109,5 @@ info:
namespace_stat:
namespace_0:
usage: 0
+ usage%: 0.0
...
diff --git a/test/box_big/limit.result b/test/box_big/limit.result
index cce04ab..90a16d2 100644
--- a/test/box_big/limit.result
+++ b/test/box_big/limit.result
@@ -31,6 +31,7 @@ An error occurred: ERR_CODE_NAMESPACE_LIMIT, 'Namespace limit
has been exceeded'
insert into t0 values (9, 'tuple')
An error occurred: ERR_CODE_NAMESPACE_LIMIT, 'Namespace limit has been
exceeded'
usage = 376
+usage% = 89.7
#
# free some space in namespace
@@ -43,11 +44,13 @@ Delete OK, 1 row affected
delete from t0 where k0 = 2
Delete OK, 1 row affected
usage = 235
+usage% = 56.1
insert into t0 values (8, 'tuple')
Insert OK, 1 row affected
insert into t0 values (9, 'tuple')
Insert OK, 1 row affected
usage = 329
+usage% = 78.5
#
# try to decrease namespace limit in runtime
@@ -93,3 +96,17 @@ Insert OK, 1 row affected
insert into t0 values (19, 'tuple')
Insert OK, 1 row affected
usage = 799
+usage% = 76.2
+
+#
+# check usage updates after replace/update
+#
+
+insert into t0 values (19, 'bigger tuple')
+Insert OK, 1 row affected
+usage = 806
+usage% = 76.9
+update t0 set k1 = 'tuple' where k0 = 19
+Update OK, 1 row affected
+usage = 799
+usage% = 76.2
diff --git a/test/box_big/limit.test b/test/box_big/limit.test
index f4dd088..8423a41 100644
--- a/test/box_big/limit.test
+++ b/test/box_big/limit.test
@@ -4,6 +4,7 @@ import yaml
def print_usage():
info = yaml.load(admin.execute("show info\n"))["info"]
print "usage = " + str(info["namespace_stat"]["namespace_0"]["usage"])
+ print "usage% = " + str(info["namespace_stat"]["namespace_0"]["usage%"])
print """
#
@@ -61,6 +62,16 @@ for i in range(10, 20):
exec sql "insert into t0 values ({0}, 'tuple')".format(i)
print_usage()
+print """
+#
+# check usage updates after replace/update
+#
+"""
+exec sql "insert into t0 values (19, 'bigger tuple')"
+print_usage()
+exec sql "update t0 set k1 = 'tuple' where k0 = 19"
+print_usage()
+
# restore default server
server.stop()
server.deploy(self.suite_ini["config"])
--
Tarantool -- an efficient key/value data store
_______________________________________________
Mailing list: https://launchpad.net/~tarantool-developers
Post to : [email protected]
Unsubscribe : https://launchpad.net/~tarantool-developers
More help : https://help.launchpad.net/ListHelp
