Hi,
> Can you change the immediate completion flag and send another?
good idea, this stripped some stuff out from the statemachines...
> The case you're probably looking for is if
> the low watermark is set to two, and the queue never gets higher than
Right, that was the case I was looking for.
Its alright, just put 2 in the patchs default settings...
Thanks,
Julian
Index: src/client/sysint/sys-small-io.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/client/sysint/sys-small-io.sm,v
retrieving revision 1.8
diff -d -r1.8 sys-small-io.sm
28,29d27
< #define SMALL_IO_MAX_REGIONS 64
<
127c125
< result.segmax = SMALL_IO_MAX_REGIONS;
---
> result.segmax = IO_MAX_REGIONS;
227,228c225,226
< PVFS_size sizes[SMALL_IO_MAX_REGIONS];
< PVFS_offset offsets[SMALL_IO_MAX_REGIONS];
---
> PVFS_size sizes[IO_MAX_REGIONS];
> PVFS_offset offsets[IO_MAX_REGIONS];
245c243
< result.segmax = SMALL_IO_MAX_REGIONS;
---
> result.segmax = IO_MAX_REGIONS;
Index: src/common/misc/mkspace.c
===================================================================
RCS file: /anoncvs/pvfs2/src/common/misc/mkspace.c,v
retrieving revision 1.38
diff -d -r1.38 mkspace.c
618c618
<
---
>
Index: src/common/misc/pvfs2-internal.h
===================================================================
RCS file: /anoncvs/pvfs2/src/common/misc/pvfs2-internal.h,v
retrieving revision 1.2
diff -d -r1.2 pvfs2-internal.h
48a49,51
>
> #define IO_MAX_REGIONS 64
>
Index: src/common/misc/server-config.c
===================================================================
RCS file: /anoncvs/pvfs2/src/common/misc/server-config.c,v
retrieving revision 1.83
diff -d -r1.83 server-config.c
80d79
< static DOTCONF_CB(get_metadata_sync_coalesce);
627,633d625
< /* This option specified that MetaData operations that have to sync
< * (setattr, etc.) should try to coallesce the sync under larger
< * workloads.
< */
< {"MetaDataSyncCoalesce", ARG_STR, get_metadata_sync_coalesce, NULL,
< CTX_STORAGEHINTS, "yes"},
<
641c633
< CTX_STORAGEHINTS, "2"},
---
> CTX_STORAGEHINTS, "1"},
1760,1781d1751
< DOTCONF_CB(get_metadata_sync_coalesce)
< {
< struct server_configuration_s *config_s =
< (struct server_configuration_s *)cmd->context;
< struct filesystem_configuration_s *fs_conf = NULL;
<
< fs_conf = (struct filesystem_configuration_s *)
< PINT_llist_head(config_s->file_systems);
<
< if(!strcmp((char *)cmd->data.str, "yes"))
< {
< fs_conf->metadata_sync_coalesce =
< (TROVE_DSPACE_SYNC_COALESCE | TROVE_KEYVAL_SYNC_COALESCE);
< }
< else
< {
< fs_conf->metadata_sync_coalesce = 0;
< }
<
< return NULL;
< }
<
3309,3330d3278
< }
<
< int PINT_config_get_trove_meta_flags(
< struct server_configuration_s *config,
< PVFS_fs_id fs_id)
< {
< int flags = 0;
< struct filesystem_configuration_s *fs_conf = NULL;
<
< if(config)
< {
< fs_conf = PINT_config_find_fs_id(config, fs_id);
< }
<
< if(fs_conf)
< {
< flags |= (fs_conf->immediate_completion ?
< TROVE_IMMEDIATE_COMPLETION : 0);
< flags |= fs_conf->metadata_sync_coalesce;
< }
<
< return flags;
Index: src/common/misc/server-config.h
===================================================================
RCS file: /anoncvs/pvfs2/src/common/misc/server-config.h,v
retrieving revision 1.50
diff -d -r1.50 server-config.h
84d83
< int metadata_sync_coalesce;
236,238d234
< struct server_configuration_s *config,
< PVFS_fs_id fs_id);
< int PINT_config_get_trove_meta_flags(
Index: src/io/trove/trove.h
===================================================================
RCS file: /anoncvs/pvfs2/src/io/trove/trove.h,v
retrieving revision 1.34
diff -d -r1.34 trove.h
62,66c62
< TROVE_IMMEDIATE_COMPLETION = 1 << 7,
< TROVE_DSPACE_SYNC_COALESCE = 1 << 8,
< TROVE_KEYVAL_SYNC_COALESCE = 1 << 9,
<
< TROVE_KEYVAL_HANDLE_COUNT = 1 << 10
---
> TROVE_KEYVAL_HANDLE_COUNT = 1 << 7
80a77,78
> TROVE_COLLECTION_META_SYNC_MODE,
> TROVE_COLLECTION_IMMEDIATE_COMPLETION,
Index: src/io/trove/trove-dbpf/dbpf-dspace.c
===================================================================
RCS file: /anoncvs/pvfs2/src/io/trove/trove-dbpf/dbpf-dspace.c,v
retrieving revision 1.135
diff -d -r1.135 dbpf-dspace.c
162c162
< return dbpf_queue_or_service(op_p, q_op_p, flags, out_op_id_p);
---
> return dbpf_queue_or_service(op_p, q_op_p, coll_p, out_op_id_p);
367c367
< return dbpf_queue_or_service(op_p, q_op_p, flags, out_op_id_p);
---
> return dbpf_queue_or_service(op_p, q_op_p, coll_p, out_op_id_p);
492c492
< return dbpf_queue_or_service(op_p, q_op_p, flags, out_op_id_p);
---
> return dbpf_queue_or_service(op_p, q_op_p, coll_p, out_op_id_p);
689c689
< return dbpf_queue_or_service(op_p, q_op_p, flags, out_op_id_p);
---
> return dbpf_queue_or_service(op_p, q_op_p, coll_p, out_op_id_p);
800c800
< return dbpf_queue_or_service(op_p, q_op_p, flags, out_op_id_p);
---
> return dbpf_queue_or_service(op_p, q_op_p, coll_p, out_op_id_p);
845c845
< return dbpf_queue_or_service(op_p, q_op_p, flags, out_op_id_p);
---
> return dbpf_queue_or_service(op_p, q_op_p, coll_p, out_op_id_p);
879c879
< op_p->coll_p->ds_db, ret, "DB->put");
---
> op_p->coll_p->ds_db, ret, "DB->put setattr");
Index: src/io/trove/trove-dbpf/dbpf-keyval.c
===================================================================
RCS file: /anoncvs/pvfs2/src/io/trove/trove-dbpf/dbpf-keyval.c,v
retrieving revision 1.73
diff -d -r1.73 dbpf-keyval.c
217c217
< return dbpf_queue_or_service(op_p, q_op_p, flags, out_op_id_p);
---
> return dbpf_queue_or_service(op_p, q_op_p, coll_p, out_op_id_p);
342c342
< return dbpf_queue_or_service(op_p, q_op_p, flags, out_op_id_p);
---
> return dbpf_queue_or_service(op_p, q_op_p, coll_p, out_op_id_p);
426c426
< if (ret != 0)
---
> if (ret != 0 )
428,429c428,430
< op_p->coll_p->keyval_db->err(
< op_p->coll_p->keyval_db, ret, "DB->put");
---
> /*op_p->coll_p->keyval_db->err(
> op_p->coll_p->keyval_db, ret, "DB->put keyval write");
> */
538c539
< return dbpf_queue_or_service(op_p, q_op_p, flags, out_op_id_p);
---
> return dbpf_queue_or_service(op_p, q_op_p, coll_p, out_op_id_p);
630c631
< return dbpf_queue_or_service(op_p, q_op_p, flags, out_op_id_p);
---
> return dbpf_queue_or_service(op_p, q_op_p, coll_p, out_op_id_p);
773c774
< return dbpf_queue_or_service(op_p, q_op_p, flags, out_op_id_p);
---
> return dbpf_queue_or_service(op_p, q_op_p, coll_p, out_op_id_p);
902c903
< return dbpf_queue_or_service(op_p, q_op_p, flags, out_op_id_p);
---
> return dbpf_queue_or_service(op_p, q_op_p, coll_p, out_op_id_p);
1005c1006
< return dbpf_queue_or_service(op_p, q_op_p, flags, out_op_id_p);
---
> return dbpf_queue_or_service(op_p, q_op_p, coll_p, out_op_id_p);
1085c1086
< op_p->coll_p->keyval_db, ret, "DB->put");
---
> op_p->coll_p->keyval_db, ret, "DB->put keyval write list");
1168c1169
< return dbpf_queue_or_service(op_p, q_op_p, flags, out_op_id_p);
---
> return dbpf_queue_or_service(op_p, q_op_p, coll_p, out_op_id_p);
1659c1660
< return dbpf_queue_or_service(op_p, q_op_p, flags, out_op_id_p);
---
> return dbpf_queue_or_service(op_p, q_op_p, coll_p, out_op_id_p);
1778c1779
< op_p->coll_p->keyval_db, ret, "DB->put");
---
> op_p->coll_p->keyval_db, ret, "DB->put keyval handle info ops");
Index: src/io/trove/trove-dbpf/dbpf-mgmt.c
===================================================================
RCS file: /anoncvs/pvfs2/src/io/trove/trove-dbpf/dbpf-mgmt.c,v
retrieving revision 1.87
diff -d -r1.87 dbpf-mgmt.c
40a41
> #include "dbpf-sync.h"
291c292,296
<
---
> struct dbpf_collection* coll;
> coll = dbpf_collection_find_registered(coll_id);
>
> assert(coll);
>
323c328
< dbpf_queued_op_set_sync_high_watermark(*(int *)parameter);
---
> dbpf_queued_op_set_sync_high_watermark(*(int *)parameter, coll);
327c332,336
< dbpf_queued_op_set_sync_low_watermark(*(int *)parameter);
---
> dbpf_queued_op_set_sync_low_watermark(*(int *)parameter, coll);
> ret = 0;
> break;
> case TROVE_COLLECTION_META_SYNC_MODE:
> dbpf_queued_op_set_sync_mode(*(int *)parameter, coll);
329a339,342
> case TROVE_COLLECTION_IMMEDIATE_COMPLETION:
> coll->immediate_completion = *(int *)parameter;
> ret = 0;
> break;
1217a1231
> gossip_debug(GOSSIP_TROVE_DEBUG, "dbpf_collection_lookup of coll: %s\n", collname);
1398a1413,1421
>
> /*
> * Initialize defaults to ensure working
> */
> coll_p->c_high_watermark = 10;
> coll_p->c_low_watermark = 1;
> coll_p->meta_sync_enabled = 1; /* MUST be 1 !*/
>
>
Index: src/io/trove/trove-dbpf/dbpf-op-queue.c
===================================================================
RCS file: /anoncvs/pvfs2/src/io/trove/trove-dbpf/dbpf-op-queue.c,v
retrieving revision 1.26
diff -d -r1.26 dbpf-op-queue.c
277c277
< if(flags & TROVE_IMMEDIATE_COMPLETION)
---
> if(coll_p->immediate_completion)
317c317
< TROVE_ds_flags flags,
---
> struct dbpf_collection *coll_p,
322c322
< if(flags & TROVE_IMMEDIATE_COMPLETION &&
---
> if( coll_p->immediate_completion &&
Index: src/io/trove/trove-dbpf/dbpf-op-queue.h
===================================================================
RCS file: /anoncvs/pvfs2/src/io/trove/trove-dbpf/dbpf-op-queue.h,v
retrieving revision 1.17
diff -d -r1.17 dbpf-op-queue.h
83c83
< TROVE_ds_flags flags,
---
> struct dbpf_collection *coll_p,
105,107d104
<
< void dbpf_queued_op_set_sync_high_watermark(int high);
< void dbpf_queued_op_set_sync_low_watermark(int low);
Index: src/io/trove/trove-dbpf/dbpf-op.h
===================================================================
RCS file: /anoncvs/pvfs2/src/io/trove/trove-dbpf/dbpf-op.h,v
retrieving revision 1.6
diff -d -r1.6 dbpf-op.h
44d43
< int coalesce_sync;
Index: src/io/trove/trove-dbpf/dbpf-sync.c
===================================================================
RCS file: /anoncvs/pvfs2/src/io/trove/trove-dbpf/dbpf-sync.c,v
retrieving revision 1.2
diff -d -r1.2 dbpf-sync.c
12,13c12,17
< static int s_high_watermark = 8;
< static int s_low_watermark = 2;
---
> enum s_sync_context_e{
> COALESCE_CONTEXT_KEYVAL,
> COALESCE_CONTEXT_DSPACE,
> COALESCE_CONTEXT_LAST,
> COALESCE_CONTEXT_NEVER_SYNC
> };
15,16c19
< static dbpf_sync_context_t * keyval_sync_array[TROVE_MAX_CONTEXTS];
< static dbpf_sync_context_t * dspace_sync_array[TROVE_MAX_CONTEXTS];
---
> static dbpf_sync_context_t sync_array[COALESCE_CONTEXT_LAST][TROVE_MAX_CONTEXTS];
22,25c25,32
< int dbpf_sync_context_init(int context_index)
< {
< keyval_sync_array[context_index] = malloc(sizeof(dbpf_sync_context_t));
< if(!keyval_sync_array[context_index])
---
> static int dbpf_sync_db(DB * dbp, enum s_sync_context_e sync_context_type, dbpf_sync_context_t * sync_context){
> int ret;
> gossip_debug(GOSSIP_DBPF_COALESCE_DEBUG,
> "[SYNC_COALESCE]:\tcoalesce %d sync start in coalesce_queue:%d pending:%d\n",
> sync_context_type, sync_context->coalesce_counter,
> sync_context->sync_counter);
> ret = dbp->sync(dbp, 0);
> if(ret != 0)
27c34,37
< return -TROVE_ENOMEM;
---
> gossip_err("db SYNC failed: %s\n",
> db_strerror(ret));
> ret = -dbpf_db_error_to_trove_error(ret);
> return ret;
28a39,43
> gossip_debug(GOSSIP_DBPF_COALESCE_DEBUG,
> "[SYNC_COALESCE]:\tcoalesce %d sync stop\n",
> sync_context_type);
> return 0;
> }
30,38c45,54
< memset(keyval_sync_array[context_index], 0, sizeof(dbpf_sync_context_t));
<
< keyval_sync_array[context_index]->mutex = gen_mutex_build();
< keyval_sync_array[context_index]->sync_queue = dbpf_op_queue_new();
<
< dspace_sync_array[context_index] = malloc(sizeof(dbpf_sync_context_t));
< if(!dspace_sync_array[context_index])
< {
< return -TROVE_ENOMEM;
---
> static int dbpf_sync_get_object_sync_context(enum dbpf_op_type type){
> if(! DBPF_OP_DOES_SYNC(type) ){
> return COALESCE_CONTEXT_NEVER_SYNC;
> }else if(DBPF_OP_IS_KEYVAL(type) ){
> return COALESCE_CONTEXT_KEYVAL;
> }else if(DBPF_OP_IS_DSPACE(type)){
> return COALESCE_CONTEXT_DSPACE;
> }else{
> //we should never get here
> assert(0);
39a56
> }
41c58,65
< memset(dspace_sync_array[context_index], 0, sizeof(dbpf_sync_context_t));
---
> int dbpf_sync_context_init(int context_index)
> {
> gossip_debug(GOSSIP_DBPF_COALESCE_DEBUG,
> "[SYNC_COALESCE]: dbpf_sync_context_init for context %d called\n",
> context_index);
> int c;
> for(c=0; c < COALESCE_CONTEXT_LAST; c++){
> bzero(& sync_array[c][context_index], sizeof(dbpf_sync_context_t));
43,44c67,69
< dspace_sync_array[context_index]->mutex = gen_mutex_build();
< dspace_sync_array[context_index]->sync_queue = dbpf_op_queue_new();
---
> sync_array[c][context_index].mutex = gen_mutex_build();
> sync_array[c][context_index].sync_queue = dbpf_op_queue_new();
> }
51,57c76,84
< gen_mutex_destroy(keyval_sync_array[context_index]->mutex);
< dbpf_op_queue_cleanup(keyval_sync_array[context_index]->sync_queue);
<
< gen_mutex_destroy(dspace_sync_array[context_index]->mutex);
< dbpf_op_queue_cleanup(dspace_sync_array[context_index]->sync_queue);
<
< free(keyval_sync_array[context_index]);
---
> gossip_debug(GOSSIP_DBPF_COALESCE_DEBUG,
> "[SYNC_COALESCE]: dbpf_sync_context_destroy for context %d called\n",
> context_index);
> int c;
> for(c=0; c < COALESCE_CONTEXT_LAST; c++){
> gen_mutex_lock(sync_array[c][context_index].mutex);
> gen_mutex_destroy(sync_array[c][context_index].mutex);
> dbpf_op_queue_cleanup(sync_array[c][context_index].sync_queue);
> }
62c89
< char * optype_string;
---
>
63a91
> DB * dbp = NULL;
65,67d92
< DB * dbp;
< int opstate = OP_SYNC_QUEUED;
<
69,131c94,145
<
< if(DBPF_OP_IS_KEYVAL(qop_p->op.type))
< {
< dbp = qop_p->op.coll_p->keyval_db;
< sync_context = keyval_sync_array[qop_p->op.context_id];
< optype_string = "keyval";
< }
< else if(DBPF_OP_IS_DSPACE(qop_p->op.type))
< {
< dbp = qop_p->op.coll_p->ds_db;
< sync_context = dspace_sync_array[qop_p->op.context_id];
< optype_string = "dspace";
< }
< else
< {
< return -TROVE_ENOSYS;
< }
<
< if(!qop_p->stats.coalesce_sync)
< {
< gossip_debug(GOSSIP_DBPF_COALESCE_DEBUG,
< "[SYNC_COALESCE]:\toperation synced "
< "individually (not coalesced)\n"
< "\t\tcoalesced: %d\n\t\tqueued: %d\n",
< sync_context->coalesce_counter,
< sync_context->sync_counter);
<
< /* do sync right now */
< gossip_debug(GOSSIP_DBPF_COALESCE_DEBUG,
< "[SYNC_COALESCE]:\tindividual %s sync start\n",
< optype_string);
< DBPF_DB_SYNC_IF_NECESSARY((&qop_p->op), dbp, ret);
< gossip_debug(GOSSIP_DBPF_COALESCE_DEBUG,
< "[SYNC_COALESCE]:\tindividual %s sync stop\n",
< optype_string);
< if(ret < 0)
< {
< return ret;
< }
<
< ret = dbpf_queued_op_complete(qop_p, 0, OP_COMPLETED);
< if(ret < 0)
< {
< return ret;
< }
<
< return 1;
< }
<
< if(!(qop_p->op.flags & TROVE_SYNC))
< {
< /* if meta sync is disabled, then we can just add the operation
< * to the completion queue.
< */
< ret = dbpf_queued_op_complete(qop_p, 0, OP_COMPLETED);
< if(ret < 0)
< {
< return ret;
< }
< opstate = OP_COMPLETED;
< }
<
< gen_mutex_lock(sync_context->mutex);
---
> enum s_sync_context_e sync_context_type = dbpf_sync_get_object_sync_context(qop_p->op.type);
> struct dbpf_collection* coll = qop_p->op.coll_p;
>
> gossip_debug(GOSSIP_DBPF_COALESCE_DEBUG,
> "[SYNC_COALESCE]: sync_coalesce called\n");
>
> if ( sync_context_type == COALESCE_CONTEXT_NEVER_SYNC )
> {
> gossip_err("Error in trove-layer, sync coalescence missused!\n");
> assert(0);
> }
>
> if ( ! (qop_p->op.flags & TROVE_SYNC) ) {
> /*
> * No sync needed at all
> */
> ret = dbpf_queued_op_complete(qop_p, 0, OP_COMPLETED);
> return 0;
> }
>
> /*
> * Now we now that this particular op is modifying
> */
> sync_context = & sync_array[sync_context_type][qop_p->op.context_id];
>
> if( sync_context_type == COALESCE_CONTEXT_DSPACE ) {
> dbp = qop_p->op.coll_p->ds_db;
> }else if( sync_context_type == COALESCE_CONTEXT_KEYVAL ) {
> dbp = qop_p->op.coll_p->keyval_db;
> }else{
> assert(0);
> }
>
> if ( ! coll->meta_sync_enabled )
> {
>
> ret = dbpf_queued_op_complete(qop_p, 0, OP_COMPLETED);
> /*
> * Sync periodical if count < lw or if lw = 0 and count > hw
> */
> int doSync=0;
>
> gen_mutex_lock(sync_context->mutex);
> sync_context->coalesce_counter++;
> if( (coll->c_high_watermark > 0 &&
> sync_context->coalesce_counter >= coll->c_high_watermark)
> || sync_context->sync_counter < coll->c_low_watermark ){
> gossip_debug(GOSSIP_DBPF_COALESCE_DEBUG,
> "[SYNC_COALESCE]:\thigh or low watermark reached:\n"
> "\t\tcoalesced: %d\n\t\tqueued: %d\n",
> sync_context->coalesce_counter,
> sync_context->sync_counter);
133,135c147,166
< if((s_high_watermark >= 0 &&
< sync_context->coalesce_counter >= s_high_watermark) ||
< sync_context->sync_counter < s_low_watermark)
---
> sync_context->coalesce_counter = 0;
> doSync = 1;
> }
> gen_mutex_unlock(sync_context->mutex);
>
> if ( doSync ) {
> ret = dbpf_sync_db(dbp, sync_context_type, sync_context);
> }
>
> return ret;
> }
>
> /*
> * metadata sync is enabled, either we delay and enqueue this op or we
> * coalesync.
> */
> gen_mutex_lock(sync_context->mutex);
> if( (sync_context->sync_counter < coll->c_low_watermark) ||
> ( coll->c_high_watermark > 0 &&
> sync_context->coalesce_counter >= coll->c_high_watermark ) )
142a174
>
144,163d175
< * because we may have already completed the operation
< * (in the case where trove meta sync is disabled).
< */
< gossip_debug(GOSSIP_DBPF_COALESCE_DEBUG,
< "[SYNC_COALESCE]:\tcoalesce %s sync start: %d\n",
< optype_string, sync_context->coalesce_counter);
< ret = dbp->sync(dbp, 0);
< if(ret != 0)
< {
< gossip_err("db SYNC failed: %s\n",
< db_strerror(ret));
< ret = -dbpf_db_error_to_trove_error(ret);
< return ret;
< }
< gossip_debug(GOSSIP_DBPF_COALESCE_DEBUG,
< "[SYNC_COALESCE]:\tcoalesce %s sync stop: %d\n",
< optype_string, sync_context->coalesce_counter);
<
< /* check if this was already moved to the completion queue
< * because trove meta sync was disabled
165,175c177,178
< if(opstate != OP_COMPLETED)
< {
< gen_mutex_lock(
< dbpf_completion_queue_array_mutex[qop_p->op.context_id]);
< dbpf_op_queue_add(
< dbpf_completion_queue_array[qop_p->op.context_id], qop_p);
< gen_mutex_lock(&qop_p->mutex);
< qop_p->op.state = OP_COMPLETED;
< qop_p->state = 0;
< gen_mutex_unlock(&qop_p->mutex);
<
---
>
> ret = dbpf_sync_db(dbp, sync_context_type, sync_context);
177,190c180,188
< /* move remaining ops in queue with ready-to-be-synced state
< * to completion queue
< */
< while(!dbpf_op_queue_empty(sync_context->sync_queue))
< {
< ready_op = dbpf_op_queue_shownext(sync_context->sync_queue);
< dbpf_op_queue_remove(ready_op);
< dbpf_op_queue_add(
< dbpf_completion_queue_array[qop_p->op.context_id],
< ready_op);
< gen_mutex_lock(&ready_op->mutex);
< ready_op->op.state = OP_COMPLETED;
< qop_p->state = 0;
< gen_mutex_unlock(&ready_op->mutex);
---
> gen_mutex_lock(
> dbpf_completion_queue_array_mutex[qop_p->op.context_id]);
>
> dbpf_op_queue_add(
> dbpf_completion_queue_array[qop_p->op.context_id], qop_p);
> gen_mutex_lock(&qop_p->mutex);
> qop_p->op.state = OP_COMPLETED;
> qop_p->state = 0;
> gen_mutex_unlock(&qop_p->mutex);
192,193d189
< sync_context->coalesce_counter--;
< }
195,199c191,194
< pthread_cond_signal(&dbpf_op_completed_cond);
< gen_mutex_unlock(
< dbpf_completion_queue_array_mutex[qop_p->op.context_id]);
< }
< else
---
> /* move remaining ops in queue with ready-to-be-synced state
> * to completion queue
> */
> while(!dbpf_op_queue_empty(sync_context->sync_queue))
201c196,204
< sync_context->coalesce_counter = 0;
---
> ready_op = dbpf_op_queue_shownext(sync_context->sync_queue);
> dbpf_op_queue_remove(ready_op);
> dbpf_op_queue_add(
> dbpf_completion_queue_array[qop_p->op.context_id],
> ready_op);
> gen_mutex_lock(&ready_op->mutex);
> ready_op->op.state = OP_COMPLETED;
> ready_op->state = 0;
> gen_mutex_unlock(&ready_op->mutex);
202a206,210
>
> sync_context->coalesce_counter = 0;
> pthread_cond_signal(&dbpf_op_completed_cond);
> gen_mutex_unlock(
> dbpf_completion_queue_array_mutex[qop_p->op.context_id]);
208,210c216,220
< "[SYNC_COALESCE]:\tcoalescing %s op: %d\n",
< optype_string, sync_context->coalesce_counter);
<
---
> "[SYNC_COALESCE]:\tcoalescing %d op: %d\n",
> sync_context_type, sync_context->coalesce_counter);
>
> dbpf_op_queue_add(
> sync_context->sync_queue, qop_p);
212,217d221
< if(opstate != OP_COMPLETED)
< {
< /* put back on the queue with SYNC_QUEUED state */
< qop_p->op.state = OP_SYNC_QUEUED;
< dbpf_op_queue_add(sync_context->sync_queue, qop_p);
< }
227d230
< char * optype_string;
228a232,233
>
> int sync_context_type = dbpf_sync_get_object_sync_context(qop_p->op.type);
233,259c238,244
< /* only perform check if operation does modifications that require sync */
< if(!DBPF_OP_DOES_SYNC(qop_p->op.type))
< {
< return 0;
< }
<
< /* only perform check if operation sync coalesce is enabled or if
< * trove meta sync is disabled, in which case we use the coalescing
< * code to perform the sync
< */
< if(DBPF_OP_IS_KEYVAL(qop_p->op.type) &&
< (qop_p->op.flags & TROVE_KEYVAL_SYNC_COALESCE))
< {
< sync_context = keyval_sync_array[qop_p->op.context_id];
< optype_string = "keyval";
< }
< else if(DBPF_OP_IS_DSPACE(qop_p->op.type) &&
< (qop_p->op.flags & TROVE_DSPACE_SYNC_COALESCE))
< {
< sync_context = dspace_sync_array[qop_p->op.context_id];
< optype_string = "dspace";
< }
< else
< {
< return 0;
< }
<
---
> if ( sync_context_type == COALESCE_CONTEXT_NEVER_SYNC )
> {
> return 0;
> }
>
> sync_context = & sync_array[sync_context_type][qop_p->op.context_id];
>
261,266c246,250
< sync_context->sync_counter++;
< if(sync_context->sync_counter >= s_low_watermark)
< {
< /* mark this queued operation as able
< * to be coalesced when syncing */
< qop_p->stats.coalesce_sync = 1;
---
>
> if( (qop_p->op.flags & TROVE_SYNC) ){
> sync_context->sync_counter++;
> }else{
> sync_context->non_sync_counter++;
268d251
<
270,272c253,255
< "[SYNC_COALESCE]: enqueue %s counter: %d\n",
< optype_string,
< sync_context->sync_counter);
---
> "[SYNC_COALESCE]: enqueue %d counter sync:%d non_sync:%d\n",
> sync_context_type,
> sync_context->sync_counter, sync_context->non_sync_counter);
282c265
< char * optype_string;
---
>
283a267,268
> int sync_context_type = dbpf_sync_get_object_sync_context(qop_p->op.type);
>
287,309c272,277
< /* only perform check if operation does modifications that require sync */
< if(!DBPF_OP_DOES_SYNC(qop_p->op.type))
< {
< return 0;
< }
<
< /* only perform check if operation sync coalesce is enabled */
< if(DBPF_OP_IS_KEYVAL(qop_p->op.type) &&
< (qop_p->op.flags & TROVE_KEYVAL_SYNC_COALESCE))
< {
< sync_context = keyval_sync_array[qop_p->op.context_id];
< optype_string = "keyval";
< }
< else if(DBPF_OP_IS_DSPACE(qop_p->op.type) &&
< (qop_p->op.flags & TROVE_DSPACE_SYNC_COALESCE))
< {
< sync_context = dspace_sync_array[qop_p->op.context_id];
< optype_string = "dspace";
< }
< else
< {
< return 0;
< }
---
> if ( sync_context_type == COALESCE_CONTEXT_NEVER_SYNC )
> {
> return 0;
> }
>
> sync_context = & sync_array[sync_context_type][qop_p->op.context_id];
312,313c280,284
< sync_context->sync_counter--;
<
---
> if( (qop_p->op.flags & TROVE_SYNC) ){
> sync_context->sync_counter--;
> }else{
> sync_context->non_sync_counter--;
> }
315,317c286,288
< "[SYNC_COALESCE]: dequeue %s counter: %d\n",
< optype_string,
< sync_context->sync_counter);
---
> "[SYNC_COALESCE]: dequeue %d counter sync:%d non_sync:%d\n",
> sync_context_type,
> sync_context->sync_counter, sync_context->non_sync_counter);
324c295
< void dbpf_queued_op_set_sync_high_watermark(int high)
---
> void dbpf_queued_op_set_sync_high_watermark(int high, struct dbpf_collection* coll)
326c297
< s_high_watermark = high;
---
> coll->c_high_watermark = high;
329c300
< void dbpf_queued_op_set_sync_low_watermark(int low)
---
> void dbpf_queued_op_set_sync_low_watermark(int low, struct dbpf_collection* coll)
331c302,310
< s_low_watermark = low;
---
> coll->c_low_watermark = low;
> }
>
> void dbpf_queued_op_set_sync_mode(int enabled, struct dbpf_collection* coll){
> coll->meta_sync_enabled = enabled;
> /*
> * Right now we don't have to check if there are operations queued in the
> * coalesync queue, because the mode is set on startup...
> */
Index: src/io/trove/trove-dbpf/dbpf-sync.h
===================================================================
RCS file: /anoncvs/pvfs2/src/io/trove/trove-dbpf/dbpf-sync.h,v
retrieving revision 1.2
diff -d -r1.2 dbpf-sync.h
13a14,19
> /*
> * Number of operations which are modifying, but are not marked with
> * TROVE_Sync
> */
> int non_sync_counter;
>
14a21
>
20a28
>
23a32,37
>
>
> void dbpf_queued_op_set_sync_high_watermark(int high, struct dbpf_collection* coll);
> void dbpf_queued_op_set_sync_low_watermark(int low, struct dbpf_collection* coll);
>
> void dbpf_queued_op_set_sync_mode(int enabled, struct dbpf_collection* coll);
Index: src/io/trove/trove-dbpf/dbpf.h
===================================================================
RCS file: /anoncvs/pvfs2/src/io/trove/trove-dbpf/dbpf.h,v
retrieving revision 1.72
diff -d -r1.72 dbpf.h
193a194,201
>
> int c_low_watermark;
> int c_high_watermark;
> int meta_sync_enabled;
> /*
> * If this option is on we don't queue ops or use threads.
> */
> int immediate_completion;
Index: src/server/chdirent.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/chdirent.sm,v
retrieving revision 1.11
diff -d -r1.11 chdirent.sm
160,161c160
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->req->u.chdirent.fs_id),
---
> 0,
193,194c192
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->req->u.chdirent.fs_id),
---
> 0,
226d223
< struct server_configuration_s *user_opts = get_server_config_struct();
255,258c252,253
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->req->u.crdirent.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->req->u.crdirent.fs_id),
---
> TROVE_SYNC |
> 0,
289d283
< struct server_configuration_s *user_opts = get_server_config_struct();
335,338c329,330
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->req->u.chdirent.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->req->u.chdirent.fs_id),
---
> TROVE_SYNC |
> 0,
Index: src/server/crdirent.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/crdirent.sm,v
retrieving revision 1.60
diff -d -r1.60 crdirent.sm
226,227c226
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->u.crdirent.fs_id),
---
> 0,
293,301c292
< /* get the flags from the config file, such as immediate completion
< * or sync coalescing.
< */
< keyval_flags = PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->u.crdirent.fs_id);
<
< /* here we add the sync flag if its specified in the config file */
< keyval_flags |= PINT_config_get_trove_sync_meta(
< get_server_config_struct(), s_op->u.crdirent.fs_id);
---
> keyval_flags = TROVE_SYNC;
351d341
< struct server_configuration_s *user_opts = get_server_config_struct();
400,403c390
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->req->u.crdirent.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->req->u.crdirent.fs_id),
---
> TROVE_SYNC,
Index: src/server/create.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/create.sm,v
retrieving revision 1.40
diff -d -r1.40 create.sm
85d84
< struct server_configuration_s *user_opts = get_server_config_struct();
92,95c91
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->req->u.create.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->req->u.create.fs_id),
---
> TROVE_SYNC ,
Index: src/server/del-eattr.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/del-eattr.sm,v
retrieving revision 1.7
diff -d -r1.7 del-eattr.sm
150d149
< struct server_configuration_s *user_opts = get_server_config_struct();
174,177c173
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->req->u.deleattr.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->req->u.deleattr.fs_id),
---
> TROVE_SYNC ,
Index: src/server/flush.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/flush.sm,v
retrieving revision 1.18
diff -d -r1.18 flush.sm
147,148c147
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->req->u.flush.fs_id),
---
> 0,
Index: src/server/get-attr.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/get-attr.sm,v
retrieving revision 1.79
diff -d -r1.79 get-attr.sm
459,460c459
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->u.getattr.fs_id),
---
> 0,
550,551c549
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->u.getattr.fs_id),
---
> 0,
610,612c608
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(),
< s_op->u.getattr.fs_id),
---
> 0,
791,792c787
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->u.getattr.fs_id),
---
> 0,
821,822c816
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->u.getattr.fs_id),
---
> 0,
Index: src/server/get-eattr.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/get-eattr.sm,v
retrieving revision 1.10
diff -d -r1.10 get-eattr.sm
210,211c210
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->req->u.geteattr.fs_id),
---
> 0,
Index: src/server/iterate-handles.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/iterate-handles.sm,v
retrieving revision 1.5
diff -d -r1.5 iterate-handles.sm
97,98c97
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->req->u.mgmt_iterate_handles.fs_id),
---
> 0,
Index: src/server/list-eattr.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/list-eattr.sm,v
retrieving revision 1.5
diff -d -r1.5 list-eattr.sm
152,153c152
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->req->u.listeattr.fs_id),
---
> 0,
Index: src/server/lookup.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/lookup.sm,v
retrieving revision 1.45
diff -d -r1.45 lookup.sm
392,393c392
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->req->u.lookup_path.fs_id),
---
> 0,
431,432c430
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->req->u.lookup_path.fs_id),
---
> 0,
Index: src/server/mgmt-get-dirdata-handle.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/mgmt-get-dirdata-handle.sm,v
retrieving revision 1.4
diff -d -r1.4 mgmt-get-dirdata-handle.sm
95,97c95
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(),
< s_op->req->u.mgmt_get_dirdata_handle.fs_id),
---
> 0,
Index: src/server/mgmt-remove-dirent.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/mgmt-remove-dirent.sm,v
retrieving revision 1.7
diff -d -r1.7 mgmt-remove-dirent.sm
96,97c96
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->req->u.mgmt_remove_dirent.fs_id),
---
> 0,
109d107
< struct server_configuration_s *user_opts = get_server_config_struct();
125,128c123
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->req->u.mgmt_remove_dirent.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->req->u.mgmt_remove_dirent.fs_id),
---
> TROVE_SYNC,
Index: src/server/mgmt-remove-object.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/mgmt-remove-object.sm,v
retrieving revision 1.8
diff -d -r1.8 mgmt-remove-object.sm
65,66c65
< struct server_configuration_s *user_opts = get_server_config_struct();
<
---
>
75,78c74
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->req->u.mgmt_remove_object.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->req->u.mgmt_remove_object.fs_id),
---
> TROVE_SYNC,
Index: src/server/mkdir.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/mkdir.sm,v
retrieving revision 1.43
diff -d -r1.43 mkdir.sm
136d135
< struct server_configuration_s *user_opts = get_server_config_struct();
150,153c149
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->u.mkdir.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->u.mkdir.fs_id),
---
> TROVE_SYNC,
176d171
< struct server_configuration_s *user_opts = get_server_config_struct();
193,196c188
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->u.mkdir.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->u.mkdir.fs_id),
---
> TROVE_SYNC,
226,229c218
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->u.mkdir.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->u.mkdir.fs_id),
---
> TROVE_SYNC,
274,275c263
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->u.mkdir.fs_id),
---
> 0,
Index: src/server/pvfs2-server.c
===================================================================
RCS file: /anoncvs/pvfs2/src/server/pvfs2-server.c,v
retrieving revision 1.217
diff -d -r1.217 pvfs2-server.c
1178a1179,1198
>
> ret = trove_collection_setinfo(
> cur_fs->coll_id, trove_context,
> TROVE_COLLECTION_META_SYNC_MODE,
> (void *)&cur_fs->trove_sync_meta);
> if(ret < 0)
> {
> gossip_lerr("Error setting coalescing low watermark\n");
> return ret;
> }
>
> ret = trove_collection_setinfo(
> cur_fs->coll_id, trove_context,
> TROVE_COLLECTION_IMMEDIATE_COMPLETION,
> (void *)&cur_fs->immediate_completion);
> if(ret < 0)
> {
> gossip_lerr("Error setting trove immediate completion\n");
> return ret;
> }
Index: src/server/pvfs2-server.h
===================================================================
RCS file: /anoncvs/pvfs2/src/server/pvfs2-server.h,v
retrieving revision 1.134
diff -d -r1.134 pvfs2-server.h
24a25
> #include "pvfs2-internal.h"
278,279d278
< #define SMALL_IO_MAX_REGIONS 64
<
282,283c281,282
< PVFS_offset offsets[SMALL_IO_MAX_REGIONS];
< PVFS_size sizes[SMALL_IO_MAX_REGIONS];
---
> PVFS_offset offsets[IO_MAX_REGIONS];
> PVFS_size sizes[IO_MAX_REGIONS];
Index: src/server/readdir.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/readdir.sm,v
retrieving revision 1.44
diff -d -r1.44 readdir.sm
143,144c143
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->req->u.readdir.fs_id),
---
> 0,
223,224c222
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->req->u.readdir.fs_id),
---
> 0,
Index: src/server/remove.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/remove.sm,v
retrieving revision 1.48
diff -d -r1.48 remove.sm
274,275c274
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->u.remove.fs_id),
---
> 0,
297,299c296
< TROVE_KEYVAL_HANDLE_COUNT |
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->u.remove.fs_id),
---
> TROVE_KEYVAL_HANDLE_COUNT,
359,362c356
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->u.remove.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->u.remove.fs_id),
---
> TROVE_SYNC,
391,394c385
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->u.remove.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->u.remove.fs_id),
---
> TROVE_SYNC,
Index: src/server/rmdirent.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/rmdirent.sm,v
retrieving revision 1.47
diff -d -r1.47 rmdirent.sm
149,150c149
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->req->u.rmdirent.fs_id),
---
> 0,
173d171
< struct server_configuration_s *user_opts = get_server_config_struct();
187,195c185,186
< /* get the flags from the config file, such as immediate completion
< * or sync coalescing.
< */
< flags = PINT_config_get_trove_meta_flags(
< user_opts, s_op->req->u.rmdirent.fs_id);
<
< /* get the sync flag from the config file */
< flags |= PINT_config_get_trove_sync_meta(
< user_opts, s_op->req->u.rmdirent.fs_id);
---
> /* set the sync flag */
> flags = TROVE_SYNC;
237d227
< struct server_configuration_s *user_opts = get_server_config_struct();
283,286c273
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->req->u.rmdirent.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->req->u.rmdirent.fs_id),
---
> TROVE_SYNC,
Index: src/server/set-attr.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/set-attr.sm,v
retrieving revision 1.63
diff -d -r1.63 set-attr.sm
261d260
< struct server_configuration_s *user_opts = get_server_config_struct();
328,331c327
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->req->u.setattr.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->req->u.setattr.fs_id),
---
> TROVE_SYNC,
381,382c377
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->req->u.setattr.fs_id),
---
> 0,
393d387
< struct server_configuration_s *user_opts = get_server_config_struct();
433,436c427
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->req->u.setattr.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->req->u.setattr.fs_id),
---
> TROVE_SYNC,
447d437
< struct server_configuration_s *user_opts = get_server_config_struct();
481,484c471
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->req->u.setattr.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->req->u.setattr.fs_id),
---
> TROVE_SYNC,
Index: src/server/set-eattr.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/set-eattr.sm,v
retrieving revision 1.10
diff -d -r1.10 set-eattr.sm
138d137
< struct server_configuration_s *user_opts = get_server_config_struct();
155,159c154
< trove_flags |=
< PINT_config_get_trove_sync_meta(
< user_opts, s_op->req->u.seteattr.fs_id) |
< PINT_config_get_trove_meta_flags(
< user_opts, s_op->req->u.seteattr.fs_id);
---
> trove_flags |= TROVE_SYNC;
Index: src/server/setparam.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/setparam.sm,v
retrieving revision 1.28
diff -d -r1.28 setparam.sm
104,105c104
< PINT_config_get_trove_meta_flags(
< get_server_config_struct(), s_op->req->u.mgmt_setparam.fs_id),
---
> 0,
Index: src/server/small-io.sm
===================================================================
RCS file: /anoncvs/pvfs2/src/server/small-io.sm,v
retrieving revision 1.15
diff -d -r1.15 small-io.sm
101c101
< result.segmax = SMALL_IO_MAX_REGIONS;
---
> result.segmax = IO_MAX_REGIONS;
_______________________________________________
Pvfs2-developers mailing list
[email protected]
http://www.beowulf-underground.org/mailman/listinfo/pvfs2-developers
