Commit: 358e07f447e9ed7f87f419e8d027e73c97c00077
Author: Campbell Barton
Date: Fri Feb 22 14:42:22 2019 +1100
Branches: master
https://developer.blender.org/rB358e07f447e9ed7f87f419e8d027e73c97c00077
readfile: reduce memory usage at load time
Delay loading all DATA sections of the blend file until they're needed.
Loading all data-blocks caused high peak memory usage especially with
libraries - since a lot of data may exist which isn't used directly.
In one test (spring project: 10_010_A.anim.blend),
peaked at ~12.5gig, dropping back to ~2.5gig once loaded.
With this change peaks memory usage reaches ~2.7gig while loading.
Besides this there are some minor gains from not having to read data
from the file-system and we can skip an alloc + memcpy reading data
written with the same version of Blender.
===================================================================
M source/blender/blenloader/intern/readfile.c
M source/blender/blenloader/intern/readfile.h
===================================================================
diff --git a/source/blender/blenloader/intern/readfile.c
b/source/blender/blenloader/intern/readfile.c
index faff680cea5..58865f0cd8d 100644
--- a/source/blender/blenloader/intern/readfile.c
+++ b/source/blender/blenloader/intern/readfile.c
@@ -223,6 +223,11 @@
* (added remark: oh, i thought that was solved? will look at that... (ton).
*/
+/**
+ * Delay reading blocks we might not use (especially applies to library
linking).
+ * which keeps large arrays in memory from data-blocks we may not even use. */
+#define USE_BHEAD_READ_ON_DEMAND
+
/* use GHash for BHead name-based lookups (speeds up linking) */
#define USE_GHASH_BHEAD
@@ -253,11 +258,22 @@ static void lib_link_animdata(FileData *fd, ID *id,
AnimData *adt);
typedef struct BHeadN {
struct BHeadN *next, *prev;
+#ifdef USE_BHEAD_READ_ON_DEMAND
+ /** Use to read the data from the file directly into memory as needed.
*/
+ int file_offset;
+ /** When set, the remainder of this allocation is the data, otherwise
it needs to be read. */
+ bool has_data;
+#endif
struct BHead bhead;
} BHeadN;
#define BHEADN_FROM_BHEAD(bh) ((BHeadN *)POINTER_OFFSET(bh, -offsetof(BHeadN,
bhead)))
+/* We could change this in the future, for now it's simplest if only data is
delayed
+ * because ID names are used in lookup tables. */
+#define BHEAD_USE_READ_ON_DEMAND(bhead) \
+ ((bhead)->code == DATA)
+
/* this function ensures that reports are printed,
* in the case of libraray linking errors this is important!
*
@@ -802,10 +818,39 @@ static BHeadN *get_bhead(FileData *fd)
/* bhead now contains the (converted) bhead structure.
Now read
* the associated data and put everything in a BHeadN
(creative naming !)
*/
- if (!fd->is_eof) {
+ if (fd->is_eof) {
+ /* pass */
+ }
+#ifdef USE_BHEAD_READ_ON_DEMAND
+ else if (fd->seek != NULL &&
BHEAD_USE_READ_ON_DEMAND(&bhead)) {
+ /* Delay reading bhead content. */
+ new_bhead = MEM_mallocN(sizeof(BHeadN),
"new_bhead");
+ if (new_bhead) {
+ new_bhead->next = new_bhead->prev =
NULL;
+ new_bhead->file_offset =
fd->file_offset;
+ new_bhead->has_data = false;
+ new_bhead->bhead = bhead;
+ int seek_new = fd->seek(fd, bhead.len,
SEEK_CUR);
+ if (seek_new == -1) {
+ fd->is_eof = true;
+ MEM_freeN(new_bhead);
+ new_bhead = NULL;
+ }
+ BLI_assert(fd->file_offset == seek_new);
+ }
+ else {
+ fd->is_eof = true;
+ }
+ }
+#endif
+ else {
new_bhead = MEM_mallocN(sizeof(BHeadN) +
bhead.len, "new_bhead");
if (new_bhead) {
new_bhead->next = new_bhead->prev =
NULL;
+#ifdef USE_BHEAD_READ_ON_DEMAND
+ new_bhead->file_offset = 0; /* don't
seek. */
+ new_bhead->has_data = true;
+#endif
new_bhead->bhead = bhead;
readsize = fd->read(fd, new_bhead + 1,
bhead.len);
@@ -887,6 +932,42 @@ BHead *blo_bhead_next(FileData *fd, BHead *thisblock)
return bhead;
}
+#ifdef USE_BHEAD_READ_ON_DEMAND
+static bool blo_bhead_read_data(FileData *fd, BHead *thisblock, void *buf)
+{
+ bool success = true;
+ BHeadN *new_bhead = BHEADN_FROM_BHEAD(thisblock);
+ BLI_assert(new_bhead->has_data == false && new_bhead->file_offset != 0);
+ int offset_backup = fd->file_offset;
+ if (UNLIKELY(fd->seek(fd, new_bhead->file_offset, SEEK_SET) == -1)) {
+ success = false;
+ }
+ else {
+ if (fd->read(fd, buf, new_bhead->bhead.len) !=
new_bhead->bhead.len) {
+ success = false;
+ }
+ }
+ if (fd->seek(fd, offset_backup, SEEK_SET) == -1) {
+ success = false;
+ }
+ return success;
+}
+
+static BHead *blo_bhead_read_full(FileData *fd, BHead *thisblock)
+{
+ BHeadN *new_bhead = BHEADN_FROM_BHEAD(thisblock);
+ BHeadN *new_bhead_data = MEM_mallocN(sizeof(BHeadN) +
new_bhead->bhead.len, "new_bhead");
+ new_bhead_data->bhead = new_bhead->bhead;
+ new_bhead_data->file_offset = new_bhead->file_offset;
+ new_bhead_data->has_data = true;
+ if (!blo_bhead_read_data(fd, thisblock, new_bhead_data + 1)) {
+ MEM_freeN(new_bhead_data);
+ return NULL;
+ }
+ return &new_bhead_data->bhead;
+}
+#endif /* USE_BHEAD_READ_ON_DEMAND */
+
/* Warning! Caller's responsibility to ensure given bhead **is** and ID one! */
const char *blo_bhead_id_name(const FileData *fd, const BHead *bhead)
{
@@ -1039,6 +1120,12 @@ static int fd_read_gzip_from_file(FileData *filedata,
void *buffer, uint size)
return (readsize);
}
+static int fd_seek_gzip_from_file(FileData *filedata, int offset, int whence)
+{
+ filedata->file_offset = gzseek(filedata->gzfiledes, offset, whence);
+ return filedata->file_offset;
+}
+
static int fd_read_from_memory(FileData *filedata, void *buffer, uint size)
{
/* don't read more bytes then there are available in the buffer */
@@ -1166,6 +1253,7 @@ FileData *blo_filedata_from_file(const char *filepath,
ReportList *reports)
FileData *fd = filedata_new();
fd->gzfiledes = gzfile;
fd->read = fd_read_gzip_from_file;
+ fd->seek = fd_seek_gzip_from_file;
/* needed for library_append and read_libraries */
BLI_strncpy(fd->relabase, filepath, sizeof(fd->relabase));
@@ -1306,8 +1394,18 @@ void blo_filedata_free(FileData *fd)
fd->buffer = NULL;
}
- // Free all BHeadN data blocks
+ /* Free all BHeadN data blocks */
+#ifndef NDEBUG
BLI_freelistN(&fd->listbase);
+#else
+ /* Sanity check we're not keeping memory we don't need. */
+ LISTBASE_FOREACH_MUTABLE (BHeadN *, new_bhead, &fd->listbase) {
+ if (fd->seek != NULL &&
BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) {
+ BLI_assert(new_bhead->has_data == 0);
+ }
+ MEM_freeN(new_bhead);
+ }
+#endif
if (fd->filesdna)
DNA_sdna_free(fd->filesdna);
@@ -1926,20 +2024,63 @@ static void *read_struct(FileData *fd, BHead *bh, const
char *blockname)
void *temp = NULL;
if (bh->len) {
+#ifdef USE_BHEAD_READ_ON_DEMAND
+ BHead *bh_orig = bh;
+ #endif
+
/* switch is based on file dna */
- if (bh->SDNAnr && (fd->flags & FD_FLAGS_SWITCH_ENDIAN))
+ if (bh->SDNAnr && (fd->flags & FD_FLAGS_SWITCH_ENDIAN)) {
+#ifdef USE_BHEAD_READ_ON_DEMAND
+ if (BHEADN_FROM_BHEAD(bh)->has_data == false) {
+ bh = blo_bhead_read_full(fd, bh);
+ if (UNLIKELY(bh == NULL)) {
+ fd->flags &= ~FD_FLAGS_FILE_OK;
+ return NULL;
+ }
+ }
+ #endif
switch_endian_structs(fd->filesdna, bh);
+ }
if (fd->compflags[bh->SDNAnr] != SDNA_CMP_REMOVED) {
if (fd->compflags[bh->SDNAnr] == SDNA_CMP_NOT_EQUAL) {
+#ifdef USE_BHEAD_READ_ON_DEMAND
+ if (BHEADN_FROM_BHEAD(bh)->has_data == false) {
+ bh = blo_bhead_read_full(fd, bh);
+ if (UNLIKELY(bh == NULL)) {
+ fd->flags &= ~FD_FLAGS_FILE_OK;
+ return NULL;
+ }
+ }
+#endif
temp = DNA_struct_reconstruct(fd->memsdna,
fd->filesdna, fd->compflags, bh->SDNAnr, bh->nr, (bh + 1));
}
else {
/* SDNA_CMP_EQUAL */
temp = MEM_mallocN(bh->len, blockname);
+#ifdef USE_BHEAD_READ_ON_DEMAND
+ if (BHEADN_FROM_BHEAD(bh)->has_data) {
+ memcpy(temp, (bh + 1), bh->len);
+ }
+ else {
+ /* Instead of allocating the bhead,
then copying it,
+ * read the data from the file directly
into the memory. */
+ if (UNLIKELY(!blo_bhead_read_data(fd,
bh, temp))) {
+ fd->flags &= ~FD_FLAGS_FILE_OK;
+ MEM_freeN(temp);
+ temp = NULL;
+ }
+ }
+#else
memcpy(temp, (bh + 1), bh->len);
+#endif
}
}
+#ifdef USE_BHEAD_READ_ON_DEMAND
+ if (bh_orig != bh) {
+ MEM_freeN(BHEADN_FROM_BHEAD(bh));
+ }
+#endif
}
return temp;
diff --git a/source/blender/blenloader/intern/readfile.h
b/source/blender/blenloader/intern/readfile.h
index 91dfae4a71d..167d5511a0c 100644
--- a/source/blender/blenloader/intern/readfile.h
+++ b/source/blender/blenloader/intern/readfile.h
@@ -56,6 +56,7 @@ typedef struct FileData {
int buffersize;
int file_offset;
int (*read)(struct FileData *filedata, void *buffer, unsigned int size);
+ int (*seek)(struct FileData *filedata, int offset, int whence);
/** Variables needed for reading from memory / stream. */
const char *buffer;
_______________________________________________
Bf-blender-cvs mailing list
[email protected]
https://lists.blender.org/mailman/listinfo/bf-blender-cvs
