From: Liu Yuan <[email protected]> Jun 08 14:32:42 queue_request(387) 1 Jun 08 14:32:42 do_io_request(105) 1, dc4435000011be , 2 Jun 08 14:32:42 do_local_io(52) 1, dc4435000011be , 2 Jun 08 14:32:42 client_rx_handler(588) connection from: 10.0.1.62:48551 Jun 08 14:32:42 queue_request(387) 1 Jun 08 14:32:42 do_io_request(105) 1, dc4435000011be , 2 Jun 08 14:32:42 do_local_io(52) 1, dc4435000011be , 2 Jun 08 14:32:42 client_rx_handler(588) connection from: 10.0.1.62:48552 Jun 08 14:32:42 queue_request(387) 1 Jun 08 14:32:42 do_io_request(105) 1, dc4435000011be , 2 Jun 08 14:32:42 do_local_io(52) 1, dc4435000011be , 2 Jun 08 14:32:42 client_rx_handler(588) connection from: 10.0.1.62:48549 Jun 08 14:32:42 queue_request(387) 1 Jun 08 14:32:42 do_io_request(105) 1, dc4435000011be , 2 Jun 08 14:32:42 client_rx_handler(588) connection from: 10.0.1.62:48550 Jun 08 14:32:42 do_local_io(52) 1, dc4435000011be , 2 Jun 08 14:32:42 queue_request(387) 2 Jun 08 14:32:42 do_io_request(105) 2, dc4435000011be , 2 Jun 08 14:32:42 do_local_io(52) 2, dc4435000011be , 2 Jun 08 14:32:42 do_io_request(111) failed: 2, dc4435000011be , 2, 3 Jun 08 14:32:42 io_op_done(119) leaving sheepdog cluster Jun 08 14:32:42 client_rx_handler(588) connection from: 10.0.1.62:48551
fix_object_consistency() might be called in multiple threads and cause trouble. So we'd remove it from the sheep core and add a manual check&repair in collie. Signed-off-by: Liu Yuan <[email protected]> --- sheep/gateway.c | 70 ------------------------------------------------ sheep/group.c | 1 - sheep/sdnet.c | 75 ---------------------------------------------------- sheep/sheep_priv.h | 2 -- 4 files changed, 148 deletions(-) diff --git a/sheep/gateway.c b/sheep/gateway.c index e92f3ed..42f028a 100644 --- a/sheep/gateway.c +++ b/sheep/gateway.c @@ -208,69 +208,6 @@ err: return ret; } -static int fix_object_consistency(struct request *req) -{ - int ret = SD_RES_NO_MEM; - unsigned int data_length; - struct sd_req *hdr = &req->rq; - struct sd_req req_bak; - struct sd_rsp rsp_bak; - void *data = req->data, *buf; - uint64_t oid = hdr->obj.oid; - int old_opcode = hdr->opcode; - - memcpy(&req_bak, &req->rq, sizeof(req_bak)); - memcpy(&rsp_bak, &req->rp, sizeof(rsp_bak)); - - if (is_vdi_obj(oid)) - data_length = SD_INODE_SIZE; - else if (is_vdi_attr_obj(oid)) - data_length = SD_ATTR_OBJ_SIZE; - else - data_length = SD_DATA_OBJ_SIZE; - - buf = valloc(data_length); - if (buf == NULL) { - eprintf("failed to allocate memory\n"); - goto out; - } - memset(buf, 0, data_length); - - - hdr->data_length = data_length; - hdr->opcode = SD_OP_READ_OBJ; - hdr->flags = 0; - hdr->obj.offset = 0; - - req->data = buf; - req->op = get_sd_op(SD_OP_READ_OBJ); - - ret = forward_read_obj_req(req); - if (ret != SD_RES_SUCCESS) { - eprintf("failed to read object %x\n", ret); - goto out; - } - - hdr->opcode = SD_OP_CREATE_AND_WRITE_OBJ; - hdr->flags = SD_FLAG_CMD_WRITE; - hdr->obj.oid = oid; - req->op = get_sd_op(hdr->opcode); - ret = forward_write_obj_req(req); - if (ret != SD_RES_SUCCESS) { - eprintf("failed to write object %x\n", ret); - goto out; - } -out: - free(buf); - req->data = data; - req->op = get_sd_op(old_opcode); - - memcpy(&req->rq, &req_bak, sizeof(req_bak)); - memcpy(&req->rp, &rsp_bak, sizeof(rsp_bak)); - - return ret; -} - void do_gateway_request(struct work *work) { struct request *req = container_of(work, struct request, work); @@ -280,12 +217,6 @@ void do_gateway_request(struct work *work) req->rq.opcode, req->rq.obj.oid, req->rq.epoch); if (!sys->enable_write_cache || bypass_object_cache(req)) { - /* fix object consistency when we read the object for the first time */ - if (req->check_consistency) { - ret = fix_object_consistency(req); - if (ret != SD_RES_SUCCESS) - goto out; - } if (req->rq.flags & SD_FLAG_CMD_WRITE) ret = forward_write_obj_req(req); else @@ -294,7 +225,6 @@ void do_gateway_request(struct work *work) ret = object_cache_handle_request(req); } -out: if (ret != SD_RES_SUCCESS) dprintf("failed: %x, %" PRIx64" , %u, %"PRIx32"\n", req->rq.opcode, req->rq.obj.oid, req->rq.epoch, ret); diff --git a/sheep/group.c b/sheep/group.c index 8fd2bf5..6721025 100644 --- a/sheep/group.c +++ b/sheep/group.c @@ -1160,7 +1160,6 @@ int create_cluster(int port, int64_t zone, int nr_vnodes, INIT_LIST_HEAD(&sys->failed_nodes); INIT_LIST_HEAD(&sys->delayed_nodes); - INIT_LIST_HEAD(&sys->consistent_obj_list); INIT_LIST_HEAD(&sys->blocking_conn_list); INIT_LIST_HEAD(&sys->wait_rw_queue); diff --git a/sheep/sdnet.c b/sheep/sdnet.c index f7eb6c8..34d65cf 100644 --- a/sheep/sdnet.c +++ b/sheep/sdnet.c @@ -39,76 +39,6 @@ static int is_access_local(struct request *req, uint64_t oid) return 0; } -static int need_consistency_check(struct request *req) -{ - struct sd_req *hdr = &req->rq; - - if (hdr->opcode != SD_OP_READ_OBJ) - /* consistency is fixed when clients read data for the - * first time */ - return 0; - - if (hdr->flags & SD_FLAG_CMD_WEAK_CONSISTENCY) - return 0; - - if (is_vdi_obj(hdr->obj.oid)) - /* only check consistency for data objects */ - return 0; - - return 1; -} - -static inline void set_consistency_check(struct request *req) -{ - uint32_t vdi_id = oid_to_vid(req->rq.obj.oid); - uint32_t idx = data_oid_to_idx(req->rq.obj.oid); - struct data_object_bmap *bmap; - - req->check_consistency = 1; - list_for_each_entry(bmap, &sys->consistent_obj_list, list) { - if (bmap->vdi_id == vdi_id) { - if (test_bit(idx, bmap->dobjs)) - req->check_consistency = 0; - break; - } - } -} - -static void check_object_consistency(struct sd_req *hdr) -{ - uint32_t vdi_id = oid_to_vid(hdr->obj.oid); - struct data_object_bmap *bmap, *n; - int nr_bmaps = 0; - - list_for_each_entry_safe(bmap, n, &sys->consistent_obj_list, list) { - nr_bmaps++; - if (bmap->vdi_id == vdi_id) { - set_bit(data_oid_to_idx(hdr->obj.oid), bmap->dobjs); - list_move_tail(&bmap->list, &sys->consistent_obj_list); - return; - } - } - - bmap = zalloc(sizeof(*bmap)); - if (bmap == NULL) { - eprintf("failed to allocate memory\n"); - return; - } - - dprintf("allocating a new object map\n"); - - bmap->vdi_id = vdi_id; - list_add_tail(&bmap->list, &sys->consistent_obj_list); - set_bit(data_oid_to_idx(hdr->obj.oid), bmap->dobjs); - if (nr_bmaps >= MAX_DATA_OBJECT_BMAPS) { - /* the first entry is the least recently used one */ - bmap = list_first_entry(&sys->consistent_obj_list, - struct data_object_bmap, list); - list_del(&bmap->list); - free(bmap); - } -} - static void io_op_done(struct work *work) { struct request *req = container_of(work, struct request, work); @@ -154,8 +84,6 @@ static void gateway_op_done(struct work *work) } break; case SD_RES_SUCCESS: - if (req->check_consistency && is_data_obj(hdr->obj.oid)) - check_object_consistency(hdr); break; } @@ -356,9 +284,6 @@ static void queue_gateway_request(struct request *req) if (request_in_recovery(req)) return; - if (need_consistency_check(req)) - set_consistency_check(req); - queue_work: req->work.fn = do_gateway_request; req->work.done = gateway_op_done; diff --git a/sheep/sheep_priv.h b/sheep/sheep_priv.h index 7a86533..9b6344b 100644 --- a/sheep/sheep_priv.h +++ b/sheep/sheep_priv.h @@ -76,7 +76,6 @@ struct request { uint64_t local_oid; struct vnode_info *vnodes; - int check_consistency; struct work work; }; @@ -125,7 +124,6 @@ struct cluster_info { DECLARE_BITMAP(vdi_inuse, SD_NR_VDIS); - struct list_head consistent_obj_list; struct list_head blocking_conn_list; int nr_copies; -- 1.7.10.2 -- sheepdog mailing list [email protected] http://lists.wpkg.org/mailman/listinfo/sheepdog
