Change size_t C32/C64 read/writes. Adapt Read_C32/Write_C32/Read_C64/Write_C64 to Read_CU32/etc for variables representing memory sizes: String, Blob, Vector and Hash sizes in particular.
Project: http://git-wip-us.apache.org/repos/asf/lucy/repo Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/5e4dcd2e Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/5e4dcd2e Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/5e4dcd2e Branch: refs/heads/master Commit: 5e4dcd2e31a80ed20e86ca931bbbc565f6781820 Parents: 17a3d60 Author: Marvin Humphrey <[email protected]> Authored: Wed Apr 20 18:16:06 2016 -0700 Committer: Marvin Humphrey <[email protected]> Committed: Wed Apr 20 19:26:48 2016 -0700 ---------------------------------------------------------------------- c/src/Lucy/Index/DocReader.c | 6 +++--- core/Lucy/Index/DocWriter.c | 12 ++++++++++-- core/Lucy/Index/TermVector.c | 4 ++-- core/Lucy/Plan/TextType.c | 3 ++- core/Lucy/Search/SortSpec.c | 3 ++- core/Lucy/Util/Freezer.c | 22 +++++++++++++++------- 6 files changed, 34 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/c/src/Lucy/Index/DocReader.c ---------------------------------------------------------------------- diff --git a/c/src/Lucy/Index/DocReader.c b/c/src/Lucy/Index/DocReader.c index 3610dfc..16e184e 100644 --- a/c/src/Lucy/Index/DocReader.c +++ b/c/src/Lucy/Index/DocReader.c @@ -40,7 +40,7 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) { Hash *const fields = Hash_new(1); int64_t start; uint32_t num_fields; - uint32_t field_name_cap = 31; + size_t field_name_cap = 31; char *field_name = (char*)MALLOCATE(field_name_cap + 1); // Get data file pointer from index, read number of fields. @@ -51,12 +51,12 @@ DefDocReader_Fetch_Doc_IMP(DefaultDocReader *self, int32_t doc_id) { // Decode stored data and build up the doc field by field. while (num_fields--) { - uint32_t field_name_len; + size_t field_name_len; Obj *value; FieldType *type; // Read field name. - field_name_len = InStream_Read_C32(dat_in); + field_name_len = InStream_Read_CU32(dat_in); if (field_name_len > field_name_cap) { field_name_cap = field_name_len; field_name = (char*)REALLOCATE(field_name, http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/core/Lucy/Index/DocWriter.c ---------------------------------------------------------------------- diff --git a/core/Lucy/Index/DocWriter.c b/core/Lucy/Index/DocWriter.c index 818f45b..ec583cc 100644 --- a/core/Lucy/Index/DocWriter.c +++ b/core/Lucy/Index/DocWriter.c @@ -119,14 +119,22 @@ DocWriter_Add_Inverted_Doc_IMP(DocWriter *self, Inverter *inverter, case FType_TEXT: { const char *buf = Str_Get_Ptr8((String*)value); size_t size = Str_Get_Size((String*)value); - OutStream_Write_C32(dat_out, size); + if (size > INT32_MAX) { + THROW(ERR, "Field %o over 2GB: %u64", field, + (uint64_t)size); + } + OutStream_Write_CU32(dat_out, (uint32_t)size); OutStream_Write_Bytes(dat_out, buf, size); break; } case FType_BLOB: { const char *buf = Blob_Get_Buf((Blob*)value); size_t size = Blob_Get_Size((Blob*)value); - OutStream_Write_C32(dat_out, size); + if (size > INT32_MAX) { + THROW(ERR, "Field %o over 2GB: %u64", field, + (uint64_t)size); + } + OutStream_Write_CU32(dat_out, (uint32_t)size); OutStream_Write_Bytes(dat_out, buf, size); break; } http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/core/Lucy/Index/TermVector.c ---------------------------------------------------------------------- diff --git a/core/Lucy/Index/TermVector.c b/core/Lucy/Index/TermVector.c index b1a3391..9c3a6f9 100644 --- a/core/Lucy/Index/TermVector.c +++ b/core/Lucy/Index/TermVector.c @@ -90,7 +90,7 @@ TV_Serialize_IMP(TermVector *self, OutStream *target) { Freezer_serialize_string(ivars->field, target); Freezer_serialize_string(ivars->text, target); - OutStream_Write_C64(target, ivars->num_pos); + OutStream_Write_CU64(target, ivars->num_pos); for (size_t i = 0; i < ivars->num_pos; i++) { OutStream_Write_C32(target, posits[i]); @@ -103,7 +103,7 @@ TermVector* TV_Deserialize_IMP(TermVector *self, InStream *instream) { String *field = Freezer_read_string(instream); String *text = Freezer_read_string(instream); - size_t num_pos = InStream_Read_C64(instream); + size_t num_pos = InStream_Read_CU64(instream); // Read positional data. int32_t *posits = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t)); http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/core/Lucy/Plan/TextType.c ---------------------------------------------------------------------- diff --git a/core/Lucy/Plan/TextType.c b/core/Lucy/Plan/TextType.c index 4b84fa8..67f64a3 100644 --- a/core/Lucy/Plan/TextType.c +++ b/core/Lucy/Plan/TextType.c @@ -101,7 +101,8 @@ TextTermStepper_Write_Key_Frame_IMP(TextTermStepper *self, String *string = (String*)CERTIFY(value, STRING); const char *buf = Str_Get_Ptr8(string); size_t size = Str_Get_Size(string); - OutStream_Write_C32(outstream, size); + // Skip size check because we're deep in the internals. + OutStream_Write_CU32(outstream, (uint32_t)size); OutStream_Write_Bytes(outstream, buf, size); S_set_value(self, value); http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/core/Lucy/Search/SortSpec.c ---------------------------------------------------------------------- diff --git a/core/Lucy/Search/SortSpec.c b/core/Lucy/Search/SortSpec.c index 9994bb4..21a6370 100644 --- a/core/Lucy/Search/SortSpec.c +++ b/core/Lucy/Search/SortSpec.c @@ -75,8 +75,9 @@ SortSpec_Get_Rules_IMP(SortSpec *self) { void SortSpec_Serialize_IMP(SortSpec *self, OutStream *target) { SortSpecIVARS *const ivars = SortSpec_IVARS(self); + // Skip size check. uint32_t num_rules = (uint32_t)Vec_Get_Size(ivars->rules); - OutStream_Write_C32(target, num_rules); + OutStream_Write_CU32(target, num_rules); for (uint32_t i = 0; i < num_rules; i++) { SortRule *rule = (SortRule*)Vec_Fetch(ivars->rules, i); SortRule_Serialize(rule, target); http://git-wip-us.apache.org/repos/asf/lucy/blob/5e4dcd2e/core/Lucy/Util/Freezer.c ---------------------------------------------------------------------- diff --git a/core/Lucy/Util/Freezer.c b/core/Lucy/Util/Freezer.c index 20e5e33..90a63c3 100644 --- a/core/Lucy/Util/Freezer.c +++ b/core/Lucy/Util/Freezer.c @@ -180,13 +180,16 @@ void Freezer_serialize_string(String *string, OutStream *outstream) { size_t size = Str_Get_Size(string); const char *buf = Str_Get_Ptr8(string); - OutStream_Write_C64(outstream, size); + if (size > INT32_MAX) { + THROW(ERR, "Can't serialize string above 2GB: %u64", (uint64_t)size); + } + OutStream_Write_CU64(outstream, size); OutStream_Write_Bytes(outstream, buf, size); } String* Freezer_deserialize_string(String *string, InStream *instream) { - size_t size = InStream_Read_C32(instream); + size_t size = InStream_Read_CU32(instream); if (size == SIZE_MAX) { THROW(ERR, "Can't deserialize SIZE_MAX bytes"); } @@ -208,13 +211,16 @@ Freezer_read_string(InStream *instream) { void Freezer_serialize_blob(Blob *blob, OutStream *outstream) { size_t size = Blob_Get_Size(blob); - OutStream_Write_C32(outstream, size); + if (size > INT32_MAX) { + THROW(ERR, "Can't serialize blob above 2GB: %u64", (uint64_t)size); + } + OutStream_Write_CU64(outstream, size); OutStream_Write_Bytes(outstream, Blob_Get_Buf(blob), size); } Blob* Freezer_deserialize_blob(Blob *blob, InStream *instream) { - size_t size = InStream_Read_C32(instream); + size_t size = InStream_Read_CU32(instream); char *buf = (char*)MALLOCATE(size); InStream_Read_Bytes(instream, buf, size); return Blob_init_steal(blob, buf, size); @@ -229,8 +235,9 @@ Freezer_read_blob(InStream *instream) { void Freezer_serialize_varray(Vector *array, OutStream *outstream) { uint32_t last_valid_tick = 0; + // Skip size check. uint32_t size = (uint32_t)Vec_Get_Size(array); - OutStream_Write_C32(outstream, size); + OutStream_Write_CU32(outstream, size); for (uint32_t i = 0; i < size; i++) { Obj *elem = Vec_Fetch(array, i); if (elem) { @@ -266,8 +273,9 @@ Freezer_read_varray(InStream *instream) { void Freezer_serialize_hash(Hash *hash, OutStream *outstream) { - uint32_t hash_size = Hash_Get_Size(hash); - OutStream_Write_C32(outstream, hash_size); + // Skip size check. + uint32_t hash_size = (uint32_t)Hash_Get_Size(hash); + OutStream_Write_CU32(outstream, hash_size); HashIterator *iter = HashIter_new(hash); while (HashIter_Next(iter)) {
