On Fri, Aug 5, 2016 at 12:49 PM, Ryan Moats <rmo...@us.ibm.com> wrote: > This patchset mimics the changes introduced in > > f199df26 (ovsdb-idl: Add partial map updates functionality.) > 010fe7ae (ovsdb-idlc.in: Autogenerate partial map updates functions.) > 7251075c (tests: Add test for partial map updates.) > > but for columns that store sets of values rather than key-value > pairs. These columns will now be able to use the OVSDB mutate > operation to transmit deltas on the wire rather than use > verify/update and transmit wait/update operations on the wire. > > Side effect of modifying the comments in the partial map update > tests. >
I was very interested in using this patch to verify a performance problem we were seeing. Our environment was testing with 400 chassis, a single (OpenStack provider) network, and 8000 ports. Before this change, running a test with ovn-scale-test (create port in NB DB, bind ports in SB DB) was taking approximately 8 hours (480 minutes). I've re-run this test with Ryan's change, and it now completes in 87 minutes. This patch has shaved almost 400 minutes off the test! I hope we can look at getting this into 2.6, because for large deployments, this is a huge performance win! Note, I have not verified the OpenStack portion of this yet, this was purely with the OVN control plane. Tested-by: Kyle Mestery <mest...@mestery.com> > Signed-off-by: Ryan Moats <rmo...@us.ibm.com> > --- > lib/automake.mk | 2 + > lib/ovsdb-idl-provider.h | 3 + > lib/ovsdb-idl.c | 390 > +++++++++++++++++++++++++++++++++++------------ > lib/ovsdb-idl.h | 6 + > lib/ovsdb-set-op.c | 170 +++++++++++++++++++++ > lib/ovsdb-set-op.h | 44 ++++++ > ovsdb/ovsdb-idlc.in | 65 +++++++- > tests/idltest.ovsschema | 30 ++++ > tests/ovsdb-idl.at | 36 +++++ > tests/test-ovsdb.c | 137 ++++++++++++++++- > 10 files changed, 776 insertions(+), 107 deletions(-) > create mode 100644 lib/ovsdb-set-op.c > create mode 100644 lib/ovsdb-set-op.h > > diff --git a/lib/automake.mk b/lib/automake.mk > index 97c83e9..30a281f 100644 > --- a/lib/automake.mk > +++ b/lib/automake.mk > @@ -187,6 +187,8 @@ lib_libopenvswitch_la_SOURCES = \ > lib/ovsdb-idl.h \ > lib/ovsdb-map-op.c \ > lib/ovsdb-map-op.h \ > + lib/ovsdb-set-op.c \ > + lib/ovsdb-set-op.h \ > lib/ovsdb-condition.h \ > lib/ovsdb-condition.c \ > lib/ovsdb-parser.c \ > diff --git a/lib/ovsdb-idl-provider.h b/lib/ovsdb-idl-provider.h > index 55ed793..64e8ec3 100644 > --- a/lib/ovsdb-idl-provider.h > +++ b/lib/ovsdb-idl-provider.h > @@ -20,6 +20,7 @@ > #include "openvswitch/list.h" > #include "ovsdb-idl.h" > #include "ovsdb-map-op.h" > +#include "ovsdb-set-op.h" > #include "ovsdb-types.h" > #include "openvswitch/shash.h" > #include "uuid.h" > @@ -39,6 +40,8 @@ struct ovsdb_idl_row { > struct hmap_node txn_node; /* Node in ovsdb_idl_txn's list. */ > unsigned long int *map_op_written; /* Bitmap of columns pending map ops. > */ > struct map_op_list **map_op_lists; /* Per-column map operations. */ > + unsigned long int *set_op_written; /* Bitmap of columns pending set ops. > */ > + struct set_op_list **set_op_lists; /* Per-column set operations. */ > > /* Tracking data */ > unsigned int change_seqno[OVSDB_IDL_CHANGE_MAX]; > diff --git a/lib/ovsdb-idl.c b/lib/ovsdb-idl.c > index d70fb10..691f3bf 100644 > --- a/lib/ovsdb-idl.c > +++ b/lib/ovsdb-idl.c > @@ -184,6 +184,7 @@ static struct ovsdb_idl_row *ovsdb_idl_row_create(struct > ovsdb_idl_table *, > static void ovsdb_idl_row_destroy(struct ovsdb_idl_row *); > static void ovsdb_idl_row_destroy_postprocess(struct ovsdb_idl *); > static void ovsdb_idl_destroy_all_map_op_lists(struct ovsdb_idl_row *); > +static void ovsdb_idl_destroy_all_set_op_lists(struct ovsdb_idl_row *); > > static void ovsdb_idl_row_parse(struct ovsdb_idl_row *); > static void ovsdb_idl_row_unparse(struct ovsdb_idl_row *); > @@ -200,6 +201,10 @@ static void ovsdb_idl_txn_add_map_op(struct > ovsdb_idl_row *, > const struct ovsdb_idl_column *, > struct ovsdb_datum *, > enum map_op_type); > +static void ovsdb_idl_txn_add_set_op(struct ovsdb_idl_row *, > + const struct ovsdb_idl_column *, > + struct ovsdb_datum *, > + enum set_op_type); > > static void ovsdb_idl_send_lock_request(struct ovsdb_idl *); > static void ovsdb_idl_send_unlock_request(struct ovsdb_idl *); > @@ -1811,7 +1816,9 @@ ovsdb_idl_row_create(struct ovsdb_idl_table *table, > const struct uuid *uuid) > row->uuid = *uuid; > row->table = table; > row->map_op_written = NULL; > - row->map_op_lists = NULL; > + row->map_op_written = NULL; > + row->set_op_lists = NULL; > + row->set_op_lists = NULL; > return row; > } > > @@ -1822,6 +1829,7 @@ ovsdb_idl_row_destroy(struct ovsdb_idl_row *row) > ovsdb_idl_row_clear_old(row); > hmap_remove(&row->table->rows, &row->hmap_node); > ovsdb_idl_destroy_all_map_op_lists(row); > + ovsdb_idl_destroy_all_set_op_lists(row); > if (ovsdb_idl_track_is_set(row->table)) { > row->change_seqno[OVSDB_IDL_CHANGE_DELETE] > = row->table->change_seqno[OVSDB_IDL_CHANGE_DELETE] > @@ -1856,6 +1864,27 @@ ovsdb_idl_destroy_all_map_op_lists(struct > ovsdb_idl_row *row) > } > > static void > +ovsdb_idl_destroy_all_set_op_lists(struct ovsdb_idl_row *row) > +{ > + if (row->set_op_written) { > + /* Clear Set Operation Lists */ > + size_t idx, n_columns; > + const struct ovsdb_idl_column *columns; > + const struct ovsdb_type *type; > + n_columns = row->table->class->n_columns; > + columns = row->table->class->columns; > + BITMAP_FOR_EACH_1 (idx, n_columns, row->set_op_written) { > + type = &columns[idx].type; > + set_op_list_destroy(row->set_op_lists[idx], type); > + } > + free(row->set_op_lists); > + bitmap_free(row->set_op_written); > + row->set_op_lists = NULL; > + row->set_op_written = NULL; > + } > +} > + > +static void > ovsdb_idl_row_destroy_postprocess(struct ovsdb_idl *idl) > { > size_t i; > @@ -2383,6 +2412,7 @@ ovsdb_idl_txn_disassemble(struct ovsdb_idl_txn *txn) > > HMAP_FOR_EACH_SAFE (row, next, txn_node, &txn->txn_rows) { > ovsdb_idl_destroy_all_map_op_lists(row); > + ovsdb_idl_destroy_all_set_op_lists(row); > if (row->old) { > if (row->written) { > ovsdb_idl_row_unparse(row); > @@ -2419,111 +2449,188 @@ ovsdb_idl_txn_extract_mutations(struct > ovsdb_idl_row *row, > size_t idx; > bool any_mutations = false; > > - BITMAP_FOR_EACH_1(idx, class->n_columns, row->map_op_written) { > - struct map_op_list *map_op_list; > - const struct ovsdb_idl_column *column; > - const struct ovsdb_datum *old_datum; > - enum ovsdb_atomic_type key_type, value_type; > - struct json *mutation, *map, *col_name, *mutator; > - struct json *del_set, *ins_map; > - bool any_del, any_ins; > - > - map_op_list = row->map_op_lists[idx]; > - column = &class->columns[idx]; > - key_type = column->type.key.type; > - value_type = column->type.value.type; > - > - /* Get the value to be changed */ > - if (row->new && row->written && bitmap_is_set(row->written,idx)) { > - old_datum = &row->new[idx]; > - } else if (row->old != NULL) { > - old_datum = &row->old[idx]; > - } else { > - old_datum = ovsdb_datum_default(&column->type); > - } > - > - del_set = json_array_create_empty(); > - ins_map = json_array_create_empty(); > - any_del = false; > - any_ins = false; > - > - for (struct map_op *map_op = map_op_list_first(map_op_list); map_op; > - map_op = map_op_list_next(map_op_list, map_op)) { > - > - if (map_op_type(map_op) == MAP_OP_UPDATE) { > - /* Find out if value really changed. */ > - struct ovsdb_datum *new_datum; > - unsigned int pos; > - new_datum = map_op_datum(map_op); > - pos = ovsdb_datum_find_key(old_datum, > - &new_datum->keys[0], > - key_type); > - if (ovsdb_atom_equals(&new_datum->values[0], > - &old_datum->values[pos], > - value_type)) { > - /* No change in value. Move on to next update. */ > - continue; > + if (row->map_op_written) { > + BITMAP_FOR_EACH_1(idx, class->n_columns, row->map_op_written) { > + struct map_op_list *map_op_list; > + const struct ovsdb_idl_column *column; > + const struct ovsdb_datum *old_datum; > + enum ovsdb_atomic_type key_type, value_type; > + struct json *mutation, *map, *col_name, *mutator; > + struct json *del_set, *ins_map; > + bool any_del, any_ins; > + > + map_op_list = row->map_op_lists[idx]; > + column = &class->columns[idx]; > + key_type = column->type.key.type; > + value_type = column->type.value.type; > + > + /* Get the value to be changed */ > + if (row->new && row->written && bitmap_is_set(row->written,idx)) > { > + old_datum = &row->new[idx]; > + } else if (row->old != NULL) { > + old_datum = &row->old[idx]; > + } else { > + old_datum = ovsdb_datum_default(&column->type); > + } > + > + del_set = json_array_create_empty(); > + ins_map = json_array_create_empty(); > + any_del = false; > + any_ins = false; > + > + for (struct map_op *map_op = map_op_list_first(map_op_list); > map_op; > + map_op = map_op_list_next(map_op_list, map_op)) { > + > + if (map_op_type(map_op) == MAP_OP_UPDATE) { > + /* Find out if value really changed. */ > + struct ovsdb_datum *new_datum; > + unsigned int pos; > + new_datum = map_op_datum(map_op); > + pos = ovsdb_datum_find_key(old_datum, > + &new_datum->keys[0], > + key_type); > + if (ovsdb_atom_equals(&new_datum->values[0], > + &old_datum->values[pos], > + value_type)) { > + /* No change in value. Move on to next update. */ > + continue; > + } > + } else if (map_op_type(map_op) == MAP_OP_DELETE){ > + /* Verify that there is a key to delete. */ > + unsigned int pos; > + pos = ovsdb_datum_find_key(old_datum, > + > &map_op_datum(map_op)->keys[0], > + key_type); > + if (pos == UINT_MAX) { > + /* No key to delete. Move on to next update. */ > + VLOG_WARN("Trying to delete a key that doesn't " > + "exist in the map."); > + continue; > + } > } > - } else if (map_op_type(map_op) == MAP_OP_DELETE){ > - /* Verify that there is a key to delete. */ > - unsigned int pos; > - pos = ovsdb_datum_find_key(old_datum, > - &map_op_datum(map_op)->keys[0], > - key_type); > - if (pos == UINT_MAX) { > - /* No key to delete. Move on to next update. */ > - VLOG_WARN("Trying to delete a key that doesn't " > - "exist in the map."); > - continue; > + > + if (map_op_type(map_op) == MAP_OP_INSERT) { > + map = json_array_create_2( > + ovsdb_atom_to_json(&map_op_datum(map_op)->keys[0], > + key_type), > + ovsdb_atom_to_json(&map_op_datum(map_op)->values[0], > + value_type)); > + json_array_add(ins_map, map); > + any_ins = true; > + } else { /* MAP_OP_UPDATE or MAP_OP_DELETE */ > + map = ovsdb_atom_to_json(&map_op_datum(map_op)->keys[0], > + key_type); > + json_array_add(del_set, map); > + any_del = true; > } > - } > > - if (map_op_type(map_op) == MAP_OP_INSERT) { > - map = json_array_create_2( > - ovsdb_atom_to_json(&map_op_datum(map_op)->keys[0], > - key_type), > - ovsdb_atom_to_json(&map_op_datum(map_op)->values[0], > - value_type)); > - json_array_add(ins_map, map); > - any_ins = true; > - } else { /* MAP_OP_UPDATE or MAP_OP_DELETE */ > - map = ovsdb_atom_to_json(&map_op_datum(map_op)->keys[0], > - key_type); > - json_array_add(del_set, map); > - any_del = true; > + /* Generate an additional insert mutate for updates. */ > + if (map_op_type(map_op) == MAP_OP_UPDATE) { > + map = json_array_create_2( > + ovsdb_atom_to_json(&map_op_datum(map_op)->keys[0], > + key_type), > + ovsdb_atom_to_json(&map_op_datum(map_op)->values[0], > + value_type)); > + json_array_add(ins_map, map); > + any_ins = true; > + } > } > > - /* Generate an additional insert mutate for updates. */ > - if (map_op_type(map_op) == MAP_OP_UPDATE) { > - map = json_array_create_2( > - ovsdb_atom_to_json(&map_op_datum(map_op)->keys[0], > - key_type), > - ovsdb_atom_to_json(&map_op_datum(map_op)->values[0], > - value_type)); > - json_array_add(ins_map, map); > - any_ins = true; > + if (any_del) { > + col_name = json_string_create(column->name); > + mutator = json_string_create("delete"); > + map = json_array_create_2(json_string_create("set"), > del_set); > + mutation = json_array_create_3(col_name, mutator, map); > + json_array_add(mutations, mutation); > + any_mutations = true; > + } else { > + json_destroy(del_set); > + } > + if (any_ins) { > + col_name = json_string_create(column->name); > + mutator = json_string_create("insert"); > + map = json_array_create_2(json_string_create("map"), > ins_map); > + mutation = json_array_create_3(col_name, mutator, map); > + json_array_add(mutations, mutation); > + any_mutations = true; > + } else { > + json_destroy(ins_map); > } > } > + } > + if (row->set_op_written) { > + BITMAP_FOR_EACH_1(idx, class->n_columns, row->set_op_written) { > + struct set_op_list *set_op_list; > + const struct ovsdb_idl_column *column; > + const struct ovsdb_datum *old_datum; > + enum ovsdb_atomic_type key_type; > + struct json *mutation, *set, *col_name, *mutator; > + struct json *del_set, *ins_set; > + bool any_del, any_ins; > + > + set_op_list = row->set_op_lists[idx]; > + column = &class->columns[idx]; > + key_type = column->type.key.type; > + > + /* Get the value to be changed */ > + if (row->new && row->written && bitmap_is_set(row->written,idx)) > { > + old_datum = &row->new[idx]; > + } else if (row->old != NULL) { > + old_datum = &row->old[idx]; > + } else { > + old_datum = ovsdb_datum_default(&column->type); > + } > > - if (any_del) { > - col_name = json_string_create(column->name); > - mutator = json_string_create("delete"); > - map = json_array_create_2(json_string_create("set"), del_set); > - mutation = json_array_create_3(col_name, mutator, map); > - json_array_add(mutations, mutation); > - any_mutations = true; > - } else { > - json_destroy(del_set); > - } > - if (any_ins) { > - col_name = json_string_create(column->name); > - mutator = json_string_create("insert"); > - map = json_array_create_2(json_string_create("map"), ins_map); > - mutation = json_array_create_3(col_name, mutator, map); > - json_array_add(mutations, mutation); > - any_mutations = true; > - } else { > - json_destroy(ins_map); > + del_set = json_array_create_empty(); > + ins_set = json_array_create_empty(); > + any_del = false; > + any_ins = false; > + > + for (struct set_op *set_op = set_op_list_first(set_op_list); > set_op; > + set_op = set_op_list_next(set_op_list, set_op)) { > + if (set_op_type(set_op) == SET_OP_INSERT) { > + set = ovsdb_atom_to_json(&set_op_datum(set_op)->keys[0], > + key_type); > + json_array_add(ins_set, set); > + any_ins = true; > + } else { /* SETP_OP_DELETE */ > + /* Verify that there is a key to delete. */ > + unsigned int pos; > + pos = ovsdb_datum_find_key(old_datum, > + > &set_op_datum(set_op)->keys[0], > + key_type); > + if (pos == UINT_MAX) { > + /* No key to delete. Move on to next update. */ > + VLOG_WARN("Trying to delete a key that doesn't " > + "exist in the set."); > + continue; > + } > + set = ovsdb_atom_to_json(&set_op_datum(set_op)->keys[0], > + key_type); > + json_array_add(del_set, set); > + any_del = true; > + } > + } > + if (any_del) { > + col_name = json_string_create(column->name); > + mutator = json_string_create("delete"); > + set = json_array_create_2(json_string_create("set"), > del_set); > + mutation = json_array_create_3(col_name, mutator, set); > + json_array_add(mutations, mutation); > + any_mutations = true; > + } else { > + json_destroy(del_set); > + } > + if (any_ins) { > + col_name = json_string_create(column->name); > + mutator = json_string_create("insert"); > + set = json_array_create_2(json_string_create("set"), > ins_set); > + mutation = json_array_create_3(col_name, mutator, set); > + json_array_add(mutations, mutation); > + any_mutations = true; > + } else { > + json_destroy(ins_set); > + } > } > } > return any_mutations; > @@ -2716,8 +2823,8 @@ ovsdb_idl_txn_commit(struct ovsdb_idl_txn *txn) > } > } > > - /* Add mutate operation, for partial map updates. */ > - if (row->map_op_written) { > + /* Add mutate operation, for partial map or partial set updates. */ > + if (row->map_op_written || row->set_op_written) { > struct json *op, *mutations; > bool any_mutations; > > @@ -3581,6 +3688,42 @@ ovsdb_idl_txn_add_map_op(struct ovsdb_idl_row *row, > } > } > > +/* Inserts a new Set Operation into current transaction. */ > +static void > +ovsdb_idl_txn_add_set_op(struct ovsdb_idl_row *row, > + const struct ovsdb_idl_column *column, > + struct ovsdb_datum *datum, > + enum set_op_type op_type) > +{ > + const struct ovsdb_idl_table_class *class; > + size_t column_idx; > + struct set_op *set_op; > + > + class = row->table->class; > + column_idx = column - class->columns; > + > + /* Check if a set operation list exists for this column. */ > + if (!row->set_op_written) { > + row->set_op_written = bitmap_allocate(class->n_columns); > + row->set_op_lists = xzalloc(class->n_columns * > + sizeof *row->set_op_lists); > + } > + if (!row->set_op_lists[column_idx]) { > + row->set_op_lists[column_idx] = set_op_list_create(); > + } > + > + /* Add a set operation to the corresponding list. */ > + set_op = set_op_create(datum, op_type); > + bitmap_set1(row->set_op_written, column_idx); > + set_op_list_add(row->set_op_lists[column_idx], set_op, &column->type); > + > + /* Add this row to the transactions's list of rows. */ > + if (hmap_node_is_null(&row->txn_node)) { > + hmap_insert(&row->table->idl->txn->txn_rows, &row->txn_node, > + uuid_hash(&row->uuid)); > + } > +} > + > static bool > is_valid_partial_update(const struct ovsdb_idl_row *row, > const struct ovsdb_idl_column *column, > @@ -3602,6 +3745,53 @@ is_valid_partial_update(const struct ovsdb_idl_row > *row, > return true; > } > > +/* Inserts the value described in 'datum' into the map in 'column' in > + * 'row_'. If the value doesn't already exist in 'column' then it's value > + * is added. The value in 'datum' must be of the same type as the values > + * in 'column'. This function takes ownership of 'datum'. > + * > + * Usually this function is used indirectly through one of the "update" > + * functions generated by vswitch-idl. */ > +void > +ovsdb_idl_txn_write_partial_set(const struct ovsdb_idl_row *row_, > + const struct ovsdb_idl_column *column, > + struct ovsdb_datum *datum) > +{ > + struct ovsdb_idl_row *row = CONST_CAST(struct ovsdb_idl_row *, row_); > + enum set_op_type op_type; > + > + if (!is_valid_partial_update(row, column, datum)) { > + ovsdb_datum_destroy(datum, &column->type); > + return; > + } > + > + op_type = SET_OP_INSERT; > + > + ovsdb_idl_txn_add_set_op(row, column, datum, op_type); > +} > + > +/* Deletes the value specified in 'datum' from the set in 'column' in 'row_'. > + * The value in 'datum' must be of the same type as the keys in 'column'. > + * This function takes ownership of 'datum'. > + * > + * Usually this function is used indirectly through one of the "update" > + * functions generated by vswitch-idl. */ > +void > +ovsdb_idl_txn_delete_partial_set(const struct ovsdb_idl_row *row_, > + const struct ovsdb_idl_column *column, > + struct ovsdb_datum *datum) > +{ > + struct ovsdb_idl_row *row = CONST_CAST(struct ovsdb_idl_row *, row_); > + > + if (!is_valid_partial_update(row, column, datum)) { > + struct ovsdb_type type_ = column->type; > + type_.value.type = OVSDB_TYPE_VOID; > + ovsdb_datum_destroy(datum, &type_); > + return; > + } > + ovsdb_idl_txn_add_set_op(row, column, datum, SET_OP_DELETE); > +} > + > /* Inserts the key-value specified in 'datum' into the map in 'column' in > * 'row_'. If the key already exist in 'column', then it's value is updated > * with the value in 'datum'. The key-value in 'datum' must be of the same > type > diff --git a/lib/ovsdb-idl.h b/lib/ovsdb-idl.h > index e25bfef..6ee6572 100644 > --- a/lib/ovsdb-idl.h > +++ b/lib/ovsdb-idl.h > @@ -276,6 +276,12 @@ void ovsdb_idl_txn_write_partial_map(const struct > ovsdb_idl_row *, > void ovsdb_idl_txn_delete_partial_map(const struct ovsdb_idl_row *, > const struct ovsdb_idl_column *, > struct ovsdb_datum *); > +void ovsdb_idl_txn_write_partial_set(const struct ovsdb_idl_row *, > + const struct ovsdb_idl_column *, > + struct ovsdb_datum *); > +void ovsdb_idl_txn_delete_partial_set(const struct ovsdb_idl_row *, > + const struct ovsdb_idl_column *, > + struct ovsdb_datum *); > void ovsdb_idl_txn_delete(const struct ovsdb_idl_row *); > const struct ovsdb_idl_row *ovsdb_idl_txn_insert( > struct ovsdb_idl_txn *, const struct ovsdb_idl_table_class *, > diff --git a/lib/ovsdb-set-op.c b/lib/ovsdb-set-op.c > new file mode 100644 > index 0000000..f41034c > --- /dev/null > +++ b/lib/ovsdb-set-op.c > @@ -0,0 +1,170 @@ > +/* Copyright (C) 2016, IBM > + * All Rights Reserved. > + * > + * Licensed under the Apache License, Version 2.0 (the "License"); you may > + * not use this file except in compliance with the License. You may obtain > + * a copy of the License at > + * > + * http://www.apache.org/licenses/LICENSE-2.0 > + * > + * Unless required by applicable law or agreed to in writing, software > + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT > + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the > + * License for the specific language governing permissions and limitations > + * under the License. > + */ > + > +#include <config.h> > +#include "ovsdb-set-op.h" > +#include "util.h" > + > +/* Set Operation: a Partial Set Update */ > +struct set_op { > + struct hmap_node node; > + struct ovsdb_datum *datum; > + enum set_op_type type; > +}; > + > +/* List of Set Operations */ > +struct set_op_list { > + struct hmap hmap; > +}; > + > +static void set_op_destroy_datum(struct set_op *, const struct ovsdb_type *); > +static struct set_op *set_op_list_find(struct set_op_list *, struct set_op *, > + const struct ovsdb_type *, size_t); > + > +struct set_op* > +set_op_create(struct ovsdb_datum *datum, enum set_op_type type) > +{ > + struct set_op *set_op = xmalloc(sizeof *set_op); > + set_op->node.hash = 0; > + set_op->node.next = HMAP_NODE_NULL; > + set_op->datum = datum; > + set_op->type = type; > + return set_op; > +} > + > +static void > +set_op_destroy_datum(struct set_op *set_op, const struct ovsdb_type *type) > +{ > + if (set_op->type == SET_OP_DELETE){ > + struct ovsdb_type type_ = *type; > + type_.value.type = OVSDB_TYPE_VOID; > + ovsdb_datum_destroy(set_op->datum, &type_); > + } else { > + ovsdb_datum_destroy(set_op->datum, type); > + } > + free(set_op->datum); > + set_op->datum = NULL; > +} > + > +void > +set_op_destroy(struct set_op *set_op, const struct ovsdb_type *type) > +{ > + set_op_destroy_datum(set_op, type); > + free(set_op); > +} > + > +struct ovsdb_datum* > +set_op_datum(const struct set_op *set_op) > +{ > + return set_op->datum; > +} > + > +enum set_op_type > +set_op_type(const struct set_op *set_op) > +{ > + return set_op->type; > +} > + > +struct set_op_list* > +set_op_list_create(void) > +{ > + struct set_op_list *list = xmalloc(sizeof *list); > + hmap_init(&list->hmap); > + return list; > +} > + > +void > +set_op_list_destroy(struct set_op_list *list, const struct ovsdb_type *type) > +{ > + struct set_op *set_op, *next; > + HMAP_FOR_EACH_SAFE (set_op, next, node, &list->hmap) { > + set_op_destroy(set_op, type); > + } > + hmap_destroy(&list->hmap); > + free(list); > +} > + > +static struct set_op* > +set_op_list_find(struct set_op_list *list, struct set_op *set_op, > + const struct ovsdb_type *type, size_t hash) > +{ > + struct set_op *found = NULL; > + struct set_op *old; > + HMAP_FOR_EACH_WITH_HASH(old, node, hash, &list->hmap) { > + if (ovsdb_atom_equals(&old->datum->keys[0], &set_op->datum->keys[0], > + type->key.type)) { > + found = old; > + break; > + } > + } > + return found; > +} > + > +/* Inserts 'set_op' into 'list'. Makes sure that any conflict with a previous > + * set operation is resolved, so only one set operation is possible on each > key > + * per transactions. 'type' must be the type of the column over which the set > + * operation will be applied. */ > +void > +set_op_list_add(struct set_op_list *list, struct set_op *set_op, > + const struct ovsdb_type *type) > +{ > + /* Check if there is a previous update with the same key. */ > + size_t hash; > + struct set_op *prev_set_op; > + > + hash = ovsdb_atom_hash(&set_op->datum->keys[0], type->key.type, 0); > + prev_set_op = set_op_list_find(list, set_op, type, hash); > + if (prev_set_op == NULL){ > + hmap_insert(&list->hmap, &set_op->node, hash); > + } else { > + if (prev_set_op->type == SET_OP_INSERT && > + set_op->type == SET_OP_DELETE) { > + /* These operations cancel each other out. */ > + hmap_remove(&list->hmap, &prev_set_op->node); > + set_op_destroy(prev_set_op, type); > + set_op_destroy(set_op, type); > + } else { > + /* For any other case, the new update operation replaces > + * the previous update operation. */ > + set_op_destroy_datum(prev_set_op, type); > + prev_set_op->type = set_op->type; > + prev_set_op->datum = set_op->datum; > + free(set_op); > + } > + } > +} > + > +struct set_op* > +set_op_list_first(struct set_op_list *list) > +{ > + struct hmap_node *node = hmap_first(&list->hmap); > + if (node == NULL) { > + return NULL; > + } > + struct set_op *set_op = CONTAINER_OF(node, struct set_op, node); > + return set_op; > +} > + > +struct set_op* > +set_op_list_next(struct set_op_list *list, struct set_op *set_op) > +{ > + struct hmap_node *node = hmap_next(&list->hmap, &set_op->node); > + if (node == NULL) { > + return NULL; > + } > + struct set_op *next = CONTAINER_OF(node, struct set_op, node); > + return next; > +} > diff --git a/lib/ovsdb-set-op.h b/lib/ovsdb-set-op.h > new file mode 100644 > index 0000000..966de28 > --- /dev/null > +++ b/lib/ovsdb-set-op.h > @@ -0,0 +1,44 @@ > +/* Copyright (C) 2016, IBM > + * All Rights Reserved. > + * > + * Licensed under the Apache License, Version 2.0 (the "License"); you may > + * not use this file except in compliance with the License. You may obtain > + * a copy of the License at > + * > + * http://www.apache.org/licenses/LICENSE-2.0 > + * > + * Unless required by applicable law or agreed to in writing, software > + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT > + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the > + * License for the specific language governing permissions and limitations > + * under the License. > + */ > + > +#ifndef OVSDB_SET_OP_H > +#define OVSDB_SET_OP_H 1 > + > +#include "ovsdb-data.h" > + > +enum set_op_type { > + SET_OP_INSERT, > + SET_OP_DELETE > +}; > + > +struct set_op; /* Set Operation: a Partial Set Update */ > +struct set_op_list; /* List of Set Operations */ > + > +/* Set Operation functions */ > +struct set_op *set_op_create(struct ovsdb_datum *, enum set_op_type); > +void set_op_destroy(struct set_op *, const struct ovsdb_type *); > +struct ovsdb_datum *set_op_datum(const struct set_op*); > +enum set_op_type set_op_type(const struct set_op*); > + > +/* Set Operation List functions */ > +struct set_op_list *set_op_list_create(void); > +void set_op_list_destroy(struct set_op_list *, const struct ovsdb_type *); > +void set_op_list_add(struct set_op_list *, struct set_op *, > + const struct ovsdb_type *); > +struct set_op *set_op_list_first(struct set_op_list *); > +struct set_op *set_op_list_next(struct set_op_list *, struct set_op *); > + > +#endif /* ovsdb-set-op.h */ > diff --git a/ovsdb/ovsdb-idlc.in b/ovsdb/ovsdb-idlc.in > index 253344e..487876b 100755 > --- a/ovsdb/ovsdb-idlc.in > +++ b/ovsdb/ovsdb-idlc.in > @@ -233,7 +233,12 @@ bool %(s)s_is_updated(const struct %(s)s *, enum > %(s)s_column_id); > print 'void %(s)s_update_%(c)s_setkey(const struct %(s)s *, > ' % {'s': structName, 'c': columnName}, > print '%(coltype)s, %(valtype)s);' % > {'coltype':column.type.key.toCType(prefix), > 'valtype':column.type.value.toCType(prefix)} > print 'void %(s)s_update_%(c)s_delkey(const struct %(s)s *, > ' % {'s': structName, 'c': columnName}, > - print '%(coltype)s);' % > {'coltype':column.type.key.toCType(prefix)}, > + print '%(coltype)s);' % > {'coltype':column.type.key.toCType(prefix)} > + if column.type.is_set(): > + print 'void %(s)s_update_%(c)s_addvalue(const struct %(s)s > *, ' % {'s': structName, 'c': columnName}, > + print '%(valtype)s);' % > {'valtype':column.type.key.toCType(prefix)} > + print 'void %(s)s_update_%(c)s_delvalue(const struct %(s)s > *, ' % {'s': structName, 'c': columnName}, > + print '%(valtype)s);' % > {'valtype':column.type.key.toCType(prefix)} > > print 'void %(s)s_add_clause_%(c)s(struct ovsdb_idl *idl, enum > ovsdb_function function,' % {'s': structName, 'c': columnName}, > if column.type.is_smap(): > @@ -854,6 +859,64 @@ void > 'valtype':column.type.value.toCType(prefix), 'S': structName.upper(), > 'C': columnName.upper()} > # End Update/Delete of partial maps > + # Update/Delete of partial set column functions > + if type.is_set(): > + print ''' > +/* Adds the value 'new_value' to the "%(c)s" set column from the "%(t)s" > table > + * in 'row'. > + * > + */ > +void > +%(s)s_update_%(c)s_addvalue(const struct %(s)s *row, %(valtype)snew_value) > +{ > + struct ovsdb_datum *datum; > + > + ovs_assert(inited); > + > + datum = xmalloc(sizeof *datum); > + datum->n = 1; > + datum->keys = xmalloc(datum->n * sizeof *datum->values); > + datum->values = NULL; > +''' % {'s': structName, 'c': columnName, > + 'valtype':column.type.key.toCType(prefix), 't': tableName} > + > + print " "+ type.key.copyCValue("datum->keys[0].%s" % > type.key.type.to_string(), "new_value") > + print ''' > + ovsdb_idl_txn_write_partial_set(&row->header_, > + &%(s)s_columns[%(S)s_COL_%(C)s], > + datum); > +}''' % {'s': structName, 'c': > columnName,'coltype':column.type.key.toCType(prefix), > + 'valtype':column.type.key.toCType(prefix), 'S': structName.upper(), > + 'C': columnName.upper()} > + print ''' > +/* Deletes the value 'delete_value' from the "%(c)s" set column from the > + * "%(t)s" table in 'row'. > + * > + */ > +void > +%(s)s_update_%(c)s_delvalue(const struct %(s)s *row, %(valtype)sdelete_value) > +{ > + struct ovsdb_datum *datum; > + > + ovs_assert(inited); > + > + datum = xmalloc(sizeof *datum); > + datum->n = 1; > + datum->keys = xmalloc(datum->n * sizeof *datum->values); > + datum->values = NULL; > +''' % {'s': structName, 'c': > columnName,'coltype':column.type.key.toCType(prefix), > + 'valtype':column.type.key.toCType(prefix), 'S': structName.upper(), > + 'C': columnName.upper(), 't': tableName} > + > + print " "+ type.key.copyCValue("datum->keys[0].%s" % > type.key.type.to_string(), "delete_value") > + print ''' > + ovsdb_idl_txn_delete_partial_set(&row->header_, > + &%(s)s_columns[%(S)s_COL_%(C)s], > + datum); > +}''' % {'s': structName, 'c': > columnName,'coltype':column.type.key.toCType(prefix), > + 'valtype':column.type.key.toCType(prefix), 'S': structName.upper(), > + 'C': columnName.upper()} > + # End Update/Delete of partial set > > # Add clause functions. > for columnName, column in sorted(table.columns.iteritems()): > diff --git a/tests/idltest.ovsschema b/tests/idltest.ovsschema > index 5482234..21d8118 100644 > --- a/tests/idltest.ovsschema > +++ b/tests/idltest.ovsschema > @@ -134,6 +134,36 @@ > } > } > } > + }, > + "simple3" : { > + "columns" : { > + "name" : { > + "type": "string" > + }, > + "uset": { > + "type": { > + "key": {"type": "uuid"}, > + "min": 0, > + "max": "unlimited" > + } > + }, > + "uref": { > + "type": { > + "key": {"type": "uuid", > + "refTable": "simple4", > + "refType": "strong"}, > + "min": 0, > + "max": "unlimited" > + } > + } > + } > + }, > + "simple4" : { > + "columns" : { > + "name" : { > + "type": "string" > + } > + } > } > } > } > diff --git a/tests/ovsdb-idl.at b/tests/ovsdb-idl.at > index b62f116..243383b 100644 > --- a/tests/ovsdb-idl.at > +++ b/tests/ovsdb-idl.at > @@ -1108,6 +1108,42 @@ OVSDB_CHECK_IDL_PARTIAL_UPDATE_MAP_COLUMN([map, > simple2 idl-partial-update-map-c > 010: End test > ]]) > > +m4_define([OVSDB_CHECK_IDL_PARTIAL_UPDATE_SET_COLUMN], > + [AT_SETUP([$1 - C]) > + AT_KEYWORDS([ovsdb server idl partial update set column positive $5]) > + AT_CHECK([ovsdb-tool create db $abs_srcdir/idltest.ovsschema], > + [0], [stdout], [ignore]) > + AT_CHECK([ovsdb-server '-vPATTERN:console:ovsdb-server|%c|%m' --detach > --no-chdir --pidfile="`pwd`"/pid --remote=punix:socket > --unixctl="`pwd`"/unixctl db], [0], [ignore], [ignore]) > + on_exit 'kill `cat pid`' > + m4_if([$2], [], [], > + [AT_CHECK([ovsdb-client transact unix:socket $2], [0], [ignore], > [ignore])]) > + AT_CHECK([test-ovsdb '-vPATTERN:console:test-ovsdb|%c|%m' -vjsonrpc -t10 > -c idl-partial-update-set-column unix:socket $3], > + [0], [stdout], [ignore]) > + AT_CHECK([sort stdout | ${PERL} $srcdir/uuidfilt.pl]m4_if([$6],,, [[| > $6]]), > + [0], [$4]) > + OVSDB_SERVER_SHUTDOWN > + AT_CLEANUP]) > + > +OVSDB_CHECK_IDL_PARTIAL_UPDATE_SET_COLUMN([set, simple3 > idl-partial-update-set-column, initially populated], > +[['["idltest", {"op":"insert", "table":"simple3", > + "row":{"name":"mySet1","uset":["set", [[ "uuid", > "0005b872-f9e5-43be-ae02-3184b9680e75" ], [ "uuid", > "000d2f6a-76af-412f-b59d-e7bcd3e84eff" ]]]} }]'] > +], > +[], > +[[000: Getting records > +001: name=mySet1 uset=[[<0>],[<1>]] uref=[] > +002: After rename+add new value > +003: name=String2 uset=[[<0>],[<1>],[<2>]] uref=[] > +004: After add new value > +005: name=String2 uset=[[<0>],[<1>],[<2>],[<3>]] uref=[] > +006: After delete value > +007: name=String2 uset=[[<0>],[<1>],[<3>]] uref=[] > +008: After trying to delete a deleted value > +009: name=String2 uset=[[<0>],[<1>],[<3>]] uref=[] > +010: After add to other table + set of strong ref > +011: name=String2 uset=[[<0>],[<1>],[<3>]] uref=[[<4>]] > +012: End test > +]]) > + > m4_define([OVSDB_CHECK_IDL_NOTIFY_PY], > [AT_SETUP([$1 - Python]) > AT_SKIP_IF([test $HAVE_PYTHON = no]) > diff --git a/tests/test-ovsdb.c b/tests/test-ovsdb.c > index 4a68bca..cb3e756 100644 > --- a/tests/test-ovsdb.c > +++ b/tests/test-ovsdb.c > @@ -208,6 +208,10 @@ usage(void) > " idl-partial-update-map-column SERVER \n" > " connect to SERVER and executes different operations to\n" > " test the capacity of updating elements inside a map column\n" > + " displaying the table information after each operation.\n" > + " idl-partial-update-set-column SERVER \n" > + " connect to SERVER and executes different operations to\n" > + " test the capacity of updating elements inside a set column\n" > " displaying the table information after each operation.\n", > program_name, program_name); > vlog_usage(); > @@ -2500,7 +2504,6 @@ dump_simple2(struct ovsdb_idl *idl, > } > } > > - > static void > do_idl_partial_update_map_column(struct ovs_cmdl_context *ctx) > { > @@ -2521,12 +2524,12 @@ do_idl_partial_update_map_column(struct > ovs_cmdl_context *ctx) > setvbuf(stdout, NULL, _IONBF, 0); > ovsdb_idl_run(idl); > > - /* Display original data in table */ > + /* Display original data in table. */ > myRow = NULL; > printf("%03d: Getting records\n", step++); > dump_simple2(idl, myRow, step++); > > - /* Insert new elements in different map columns */ > + /* Insert new elements in different map columns. */ > myRow = idltest_simple2_first(idl); > myTxn = ovsdb_idl_txn_create(idl); > idltest_simple2_get_smap(myRow, OVSDB_TYPE_STRING, > @@ -2542,7 +2545,7 @@ do_idl_partial_update_map_column(struct > ovs_cmdl_context *ctx) > printf("%03d: After insert element\n", step++); > dump_simple2(idl, myRow, step++); > > - /* Insert duplicate element */ > + /* Insert duplicate element. */ > myTxn = ovsdb_idl_txn_create(idl); > idltest_simple2_update_smap_setkey(myRow, "key1", "myList1"); > ovsdb_idl_txn_commit_block(myTxn); > @@ -2551,7 +2554,7 @@ do_idl_partial_update_map_column(struct > ovs_cmdl_context *ctx) > printf("%03d: After insert duplicated element\n", step++); > dump_simple2(idl, myRow, step++); > > - /* deletes an element of a map column */ > + /* Deletes an element of a map column. */ > myRow = idltest_simple2_first(idl); > myTxn = ovsdb_idl_txn_create(idl); > smap = idltest_simple2_get_smap(myRow, OVSDB_TYPE_STRING, > @@ -2564,7 +2567,7 @@ do_idl_partial_update_map_column(struct > ovs_cmdl_context *ctx) > printf("%03d: After delete element\n", step++); > dump_simple2(idl, myRow, step++); > > - /* try to delete a deleted element of a map column */ > + /* Try to delete a deleted element of a map column. */ > myTxn = ovsdb_idl_txn_create(idl); > idltest_simple2_update_smap_delkey(myRow, key_to_delete); > ovsdb_idl_txn_commit_block(myTxn); > @@ -2577,6 +2580,126 @@ do_idl_partial_update_map_column(struct > ovs_cmdl_context *ctx) > return; > } > > +static void > +print_idl_row_simple3(const struct idltest_simple3 *s, int step) > +{ > + size_t i; > + const struct ovsdb_datum *uset; > + const struct ovsdb_datum *uref; > + > + uset = idltest_simple3_get_uset(s, OVSDB_TYPE_UUID); > + printf("%03d: name=%s uset=[", > + step, s->name); > + for (i = 0; i < uset->n; i++) { > + printf("["UUID_FMT"]%s", UUID_ARGS(&(uset->keys[i].uuid)), i < > uset->n-1? ",": ""); > + } > + uref = idltest_simple3_get_uref(s, OVSDB_TYPE_UUID); > + printf("] uref=["); > + for (i = 0; i < uref->n; i++) { > + printf("["UUID_FMT"]%s", UUID_ARGS(&(uref->keys[i].uuid)), i < > uref->n-1? ",": ""); > + } > + printf("]\n"); > +} > + > +static void > +dump_simple3(struct ovsdb_idl *idl, > + const struct idltest_simple3 *myRow, > + int step) > +{ > + IDLTEST_SIMPLE3_FOR_EACH(myRow, idl) { > + print_idl_row_simple3(myRow, step); > + } > +} > + > +static void > +do_idl_partial_update_set_column(struct ovs_cmdl_context *ctx) > +{ > + struct ovsdb_idl *idl; > + struct ovsdb_idl_txn *myTxn; > + const struct idltest_simple3 *myRow; > + struct idltest_simple4 *myRow2; > + const struct ovsdb_datum *uset OVS_UNUSED; > + const struct ovsdb_datum *uref OVS_UNUSED; > + int step = 0; > + > + idltest_init(); > + idl = ovsdb_idl_create(ctx->argv[1], &idltest_idl_class, false, true); > + ovsdb_idl_add_table(idl, &idltest_table_simple3); > + ovsdb_idl_add_column(idl, &idltest_simple3_col_name); > + ovsdb_idl_add_column(idl, &idltest_simple3_col_uset); > + ovsdb_idl_add_column(idl, &idltest_simple3_col_uref); > + ovsdb_idl_add_table(idl, &idltest_table_simple4); > + ovsdb_idl_add_column(idl, &idltest_simple4_col_name); > + ovsdb_idl_get_initial_snapshot(idl); > + setvbuf(stdout, NULL, _IONBF, 0); > + ovsdb_idl_run(idl); > + > + /* Display original data in table. */ > + myRow = NULL; > + printf("%03d: Getting records\n", step++); > + dump_simple3(idl, myRow, step++); > + > + /* Insert new elements in different map columns. */ > + myRow = idltest_simple3_first(idl); > + myTxn = ovsdb_idl_txn_create(idl); > + idltest_simple3_get_uset(myRow, OVSDB_TYPE_UUID); > + struct uuid uuid_to_add; > + uuid_from_string(&uuid_to_add, "001e43d2-dd3f-4616-ab6a-83a490bb0991"); > + idltest_simple3_update_uset_addvalue(myRow, uuid_to_add); > + idltest_simple3_set_name(myRow, "String2"); > + ovsdb_idl_txn_commit_block(myTxn); > + ovsdb_idl_txn_destroy(myTxn); > + ovsdb_idl_get_initial_snapshot(idl); > + printf("%03d: After rename+add new value\n", step++); > + dump_simple3(idl, myRow, step++); > + > + /* Insert duplicate element. */ > + myTxn = ovsdb_idl_txn_create(idl); > + struct uuid uuid_to_add2; > + uuid_from_string(&uuid_to_add2, "0026b3ba-571b-4729-8227-d860a5210ab8"); > + idltest_simple3_update_uset_addvalue(myRow, uuid_to_add2); > + ovsdb_idl_txn_commit_block(myTxn); > + ovsdb_idl_txn_destroy(myTxn); > + ovsdb_idl_get_initial_snapshot(idl); > + printf("%03d: After add new value\n", step++); > + dump_simple3(idl, myRow, step++); > + > + /* Deletes an element of a set column. */ > + myRow = idltest_simple3_first(idl); > + myTxn = ovsdb_idl_txn_create(idl); > + uset = idltest_simple3_get_uset(myRow, OVSDB_TYPE_UUID); > + idltest_simple3_update_uset_delvalue(myRow, uuid_to_add); > + ovsdb_idl_txn_commit_block(myTxn); > + ovsdb_idl_txn_destroy(myTxn); > + ovsdb_idl_get_initial_snapshot(idl); > + printf("%03d: After delete value\n", step++); > + dump_simple3(idl, myRow, step++); > + > + /* Try to delete a deleted element of a map column. */ > + myRow = idltest_simple3_first(idl); > + myTxn = ovsdb_idl_txn_create(idl); > + idltest_simple3_update_uset_delvalue(myRow, uuid_to_add); > + ovsdb_idl_txn_commit_block(myTxn); > + ovsdb_idl_txn_destroy(myTxn); > + ovsdb_idl_get_initial_snapshot(idl); > + printf("%03d: After trying to delete a deleted value\n", step++); > + dump_simple3(idl, myRow, step++); > + > + /* Adds to a table and update a strong reference in another table. */ > + myRow = idltest_simple3_first(idl); > + myTxn = ovsdb_idl_txn_create(idl); > + myRow2 = idltest_simple4_insert(myTxn); > + idltest_simple4_set_name(myRow2, "test"); > + idltest_simple3_update_uref_addvalue(myRow, myRow2); > + ovsdb_idl_txn_commit_block(myTxn); > + ovsdb_idl_txn_destroy(myTxn); > + ovsdb_idl_get_initial_snapshot(idl); > + printf("%03d: After add to other table + set of strong ref\n", step++); > + dump_simple3(idl, myRow, step++); > + printf("%03d: End test\n", step); > + return; > +} > + > static struct ovs_cmdl_command all_commands[] = { > { "log-io", NULL, 2, INT_MAX, do_log_io }, > { "default-atoms", NULL, 0, 0, do_default_atoms }, > @@ -2609,6 +2732,8 @@ static struct ovs_cmdl_command all_commands[] = { > { "idl", NULL, 1, INT_MAX, do_idl }, > { "idl-partial-update-map-column", NULL, 1, INT_MAX, > do_idl_partial_update_map_column }, > + { "idl-partial-update-set-column", NULL, 1, INT_MAX, > + do_idl_partial_update_set_column }, > { "help", NULL, 0, INT_MAX, do_help }, > { NULL, NULL, 0, 0, NULL }, > }; > -- > 2.7.4 > > _______________________________________________ > dev mailing list > dev@openvswitch.org > http://openvswitch.org/mailman/listinfo/dev _______________________________________________ dev mailing list dev@openvswitch.org http://openvswitch.org/mailman/listinfo/dev