PR #24402 opened by Kacper Michajłow (kasper93) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24402 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24402.patch
From dd061e628f4aff8852851985fbbf38170d6cf985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Michaj=C5=82ow?= <[email protected]> Date: Mon, 7 Sep 2026 00:49:19 +0200 Subject: [PATCH 1/2] avutil/file: add av_file_map_shared() and av_file_unmap_shared() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit av_file_map() maps a private copy of a file by name. Code that shares a file between processes needs the opposite, a writable mapping backed by the file, of a descriptor it opened itself, that grows with the file. Provide it with mmap() on POSIX and with a file mapping object on Windows, which also extends the file to the requested size. Signed-off-by: Kacper Michajłow <[email protected]> --- doc/APIchanges | 3 ++ libavutil/file.c | 109 +++++++++++++++++++++++++++++++------------- libavutil/file.h | 29 ++++++++++++ libavutil/version.h | 2 +- 4 files changed, 110 insertions(+), 33 deletions(-) diff --git a/doc/APIchanges b/doc/APIchanges index 2f6a3fc08e..9c9dbd85af 100644 --- a/doc/APIchanges +++ b/doc/APIchanges @@ -2,6 +2,9 @@ The last version increases of all libraries were on 2026-06-23. API changes, most recent first: +2026-09-xx - xxxxxxxxxx - lavu 61.8.100 - file.h + Add av_file_map_shared() and av_file_unmap_shared(). + 2026-09-xx - xxxxxxxxxx - lavu 61.7.100 - samplefmt.h Add AV_SAMPLE_FMT_DSD. diff --git a/libavutil/file.c b/libavutil/file.c index 4ef940a6c3..3c082d6f3c 100644 --- a/libavutil/file.c +++ b/libavutil/file.c @@ -52,6 +52,57 @@ static const AVClass file_log_ctx_class = { .parent_log_context_offset = offsetof(FileLogContext, log_ctx), }; +#if HAVE_MMAP || HAVE_MAPVIEWOFFILE +/* Map the first size bytes of the file, as a private copy or shared and + * writable. The shared mapping extends the file to size, on Windows the + * mapping object does it. */ +static int map_file(int fd, size_t size, int shared, void **ptr) +{ +#if HAVE_MMAP + if (shared) { + struct stat st; + if (fstat(fd, &st) < 0) + return AVERROR(errno); + if (st.st_size < size && ftruncate(fd, size) < 0) + return AVERROR(errno); + } + + void *map = mmap(NULL, size, PROT_READ | PROT_WRITE, + shared ? MAP_SHARED : MAP_PRIVATE, fd, 0); + if (map == MAP_FAILED) + return AVERROR(errno); +#else + HANDLE fh = (HANDLE)_get_osfhandle(fd); + if (fh == INVALID_HANDLE_VALUE) + return AVERROR(EBADF); + + HANDLE mh = CreateFileMapping(fh, NULL, + shared ? PAGE_READWRITE : PAGE_READONLY, + (uint64_t)size >> 32, size, NULL); + if (!mh) + return AVERROR(EIO); + + void *map = MapViewOfFile(mh, shared ? FILE_MAP_ALL_ACCESS : FILE_MAP_COPY, + 0, 0, size); + CloseHandle(mh); + if (!map) + return AVERROR(EIO); +#endif + + *ptr = map; + return 0; +} + +static void unmap_file(void *ptr, size_t size) +{ +#if HAVE_MMAP + munmap(ptr, size); +#else + UnmapViewOfFile(ptr); +#endif +} +#endif + int av_file_map(const char *filename, uint8_t **bufptr, size_t *size, int log_offset, void *log_ctx) { @@ -90,39 +141,15 @@ int av_file_map(const char *filename, uint8_t **bufptr, size_t *size, goto out; } -#if HAVE_MMAP - ptr = mmap(NULL, *size, PROT_READ|PROT_WRITE, MAP_PRIVATE, fd, 0); - if (ptr == MAP_FAILED) { - err = AVERROR(errno); - av_log(&file_log_ctx, AV_LOG_ERROR, "Error occurred in mmap(): %s\n", av_err2str(err)); +#if HAVE_MMAP || HAVE_MAPVIEWOFFILE + err = map_file(fd, *size, 0, &ptr); + if (err < 0) { + av_log(&file_log_ctx, AV_LOG_ERROR, "Cannot map file '%s': %s\n", filename, av_err2str(err)); close(fd); *size = 0; return err; } *bufptr = ptr; -#elif HAVE_MAPVIEWOFFILE - { - HANDLE mh, fh = (HANDLE)_get_osfhandle(fd); - - mh = CreateFileMapping(fh, NULL, PAGE_READONLY, 0, 0, NULL); - if (!mh) { - av_log(&file_log_ctx, AV_LOG_ERROR, "Error occurred in CreateFileMapping()\n"); - close(fd); - *size = 0; - return -1; - } - - ptr = MapViewOfFile(mh, FILE_MAP_COPY, 0, 0, *size); - CloseHandle(mh); - if (!ptr) { - av_log(&file_log_ctx, AV_LOG_ERROR, "Error occurred in MapViewOfFile()\n"); - close(fd); - *size = 0; - return -1; - } - - *bufptr = ptr; - } #else *bufptr = av_malloc(*size); if (!*bufptr) { @@ -143,11 +170,29 @@ void av_file_unmap(uint8_t *bufptr, size_t size) { if (!size || !bufptr) return; -#if HAVE_MMAP - munmap(bufptr, size); -#elif HAVE_MAPVIEWOFFILE - UnmapViewOfFile(bufptr); +#if HAVE_MMAP || HAVE_MAPVIEWOFFILE + unmap_file(bufptr, size); #else av_free(bufptr); #endif } + +int av_file_map_shared(int fd, size_t size, void **bufptr) +{ + *bufptr = NULL; + if (!size) + return AVERROR(EINVAL); +#if HAVE_MMAP || HAVE_MAPVIEWOFFILE + return map_file(fd, size, 1, bufptr); +#else + return AVERROR(ENOSYS); +#endif +} + +void av_file_unmap_shared(void *bufptr, size_t size) +{ +#if HAVE_MMAP || HAVE_MAPVIEWOFFILE + if (size && bufptr) + unmap_file(bufptr, size); +#endif +} diff --git a/libavutil/file.h b/libavutil/file.h index fced170108..d542a6fd56 100644 --- a/libavutil/file.h +++ b/libavutil/file.h @@ -59,4 +59,33 @@ int av_file_map(const char *filename, uint8_t **bufptr, size_t *size, */ void av_file_unmap(uint8_t *bufptr, size_t size); +/** + * Map the beginning of an open file into memory for shared read and write + * access. + * + * Unlike av_file_map() the mapping is not a private copy of the file, every + * store to the returned memory is written to the file and is visible to + * every other mapping of it, in this process and in other processes. The + * file is extended to size bytes if it is shorter, it is never shortened. + * The mapping must be released with av_file_unmap_shared(). + * + * @param fd file descriptor of a file opened for reading and writing + * @param size number of bytes to map, must not be zero + * @param[out] bufptr pointee is set to the mapped memory + * @return 0 in case of success, a negative value corresponding to an + * AVERROR error code in case of failure, AVERROR(ENOSYS) when the platform + * has no shared file mappings + */ +av_warn_unused_result +int av_file_map_shared(int fd, size_t size, void **bufptr); + +/** + * Unmap the memory mapped by av_file_map_shared(). + * + * @param bufptr the memory previously mapped by av_file_map_shared() + * @param size size in bytes of the mapping, must be the same as passed + * to av_file_map_shared() + */ +void av_file_unmap_shared(void *bufptr, size_t size); + #endif /* AVUTIL_FILE_H */ diff --git a/libavutil/version.h b/libavutil/version.h index b83c74d755..a05c516c3d 100644 --- a/libavutil/version.h +++ b/libavutil/version.h @@ -79,7 +79,7 @@ */ #define LIBAVUTIL_VERSION_MAJOR 61 -#define LIBAVUTIL_VERSION_MINOR 7 +#define LIBAVUTIL_VERSION_MINOR 8 #define LIBAVUTIL_VERSION_MICRO 100 #define LIBAVUTIL_VERSION_INT AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \ -- 2.52.0 From b6fc95df719dc0030f3068cf3e5bfeeb7a99bc4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20Michaj=C5=82ow?= <[email protected]> Date: Mon, 7 Sep 2026 00:49:19 +0200 Subject: [PATCH 2/2] avformat/shared: use av_file_map_shared() and drop the mmap dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes this available on Windows too. The mmap and unistd_h dependencies are gone, the protocol builds everywhere and av_file_map_shared() reports the platforms without shared mappings at runtime. The cache file is no longer shortened when it is longer than the known file size, the mapping simply covers its beginning. Signed-off-by: Kacper Michajłow <[email protected]> --- configure | 2 +- libavformat/shared.c | 169 +++++++++++++++++++++++++++++-------------- 2 files changed, 114 insertions(+), 57 deletions(-) diff --git a/configure b/configure index c025dbc95d..3029e06b9d 100755 --- a/configure +++ b/configure @@ -4131,7 +4131,7 @@ schannel_conflict="openssl gnutls libtls mbedtls" sctp_protocol_deps="struct_sctp_event_subscribe struct_msghdr_msg_flags" sctp_protocol_select="network" securetransport_conflict="openssl gnutls libtls mbedtls" -shared_protocol_deps="mmap stdatomic unistd_h" +shared_protocol_deps="stdatomic" srtp_protocol_select="rtp_protocol srtp" tcp_protocol_select="network" tls_protocol_deps_any="gnutls openssl schannel securetransport libtls mbedtls" diff --git a/libavformat/shared.c b/libavformat/shared.c index 646976f507..7a0d2b3d69 100644 --- a/libavformat/shared.c +++ b/libavformat/shared.c @@ -26,6 +26,7 @@ #include "libavutil/avstring.h" #include "libavutil/crc.h" #include "libavutil/error.h" +#include "libavutil/file.h" #include "libavutil/hash.h" #include "libavutil/file_open.h" #include "libavutil/mem.h" @@ -33,6 +34,7 @@ #include "libavutil/time.h" #include "internal.h" +#include "os_support.h" #include "url.h" #include <assert.h> @@ -41,10 +43,83 @@ #include <inttypes.h> #include <stdatomic.h> #include <string.h> -#include <sys/file.h> -#include <sys/mman.h> #include <sys/stat.h> +#if HAVE_UNISTD_H #include <unistd.h> +#endif +#ifdef _WIN32 +#include <io.h> +#include <windows.h> +#else +#include <sys/file.h> +#endif + +#ifndef O_BINARY +#define O_BINARY 0 +#endif + +/* + * The cache file is read and written at offsets while it is not mapped, the + * space map file is locked while it grows. + */ +#ifdef _WIN32 +static int file_lock(int fd) +{ + OVERLAPPED ov = { 0 }; + if (!LockFileEx((HANDLE)_get_osfhandle(fd), LOCKFILE_EXCLUSIVE_LOCK, 0, + 1, 0, &ov)) + return AVERROR(EIO); + return 0; +} + +static int file_unlock(int fd) +{ + OVERLAPPED ov = { 0 }; + if (!UnlockFileEx((HANDLE)_get_osfhandle(fd), 0, 1, 0, &ov)) + return AVERROR(EIO); + return 0; +} + +static int file_pread(int fd, void *buf, size_t size, int64_t offset) +{ + OVERLAPPED ov = { .Offset = offset, .OffsetHigh = offset >> 32 }; + DWORD n; + if (!ReadFile((HANDLE)_get_osfhandle(fd), buf, size, &n, &ov)) + return GetLastError() == ERROR_HANDLE_EOF ? 0 : AVERROR(EIO); + return n; +} + +static int file_pwrite(int fd, const void *buf, size_t size, int64_t offset) +{ + OVERLAPPED ov = { .Offset = offset, .OffsetHigh = offset >> 32 }; + DWORD n; + if (!WriteFile((HANDLE)_get_osfhandle(fd), buf, size, &n, &ov)) + return AVERROR(EIO); + return n; +} +#else +static int file_lock(int fd) +{ + return flock(fd, LOCK_EX) < 0 ? AVERROR(errno) : 0; +} + +static int file_unlock(int fd) +{ + return flock(fd, LOCK_UN) < 0 ? AVERROR(errno) : 0; +} + +static int file_pread(int fd, void *buf, size_t size, int64_t offset) +{ + ssize_t ret = pread(fd, buf, size, offset); + return ret < 0 ? AVERROR(errno) : ret; +} + +static int file_pwrite(int fd, const void *buf, size_t size, int64_t offset) +{ + ssize_t ret = pwrite(fd, buf, size, offset); + return ret < 0 ? AVERROR(errno) : ret; +} +#endif /** * This hash should be resistant against collision attacks, so that an @@ -175,15 +250,15 @@ typedef struct SharedContext { int64_t filesize; ///< once known /* cache file */ - uint8_t *cache_data; ///< optional mmap of the cache file + uint8_t *cache_data; ///< optional mapping of the cache file char *cache_path; - off_t cache_size; ///< size of mapped memory region (for munmap) + size_t cache_size; ///< size of the mapping int fd; /* space map */ Spacemap *spacemap; char *map_path; - off_t map_size; + size_t map_size; int mapfd; /* statistics */ @@ -196,10 +271,8 @@ static int shared_close(URLContext *h) SharedContext *s = h->priv_data; ffurl_close(s->inner); - if (s->cache_data) - munmap(s->cache_data, s->cache_size); - if (s->spacemap) - munmap(s->spacemap, s->map_size); + av_file_unmap_shared(s->cache_data, s->cache_size); + av_file_unmap_shared(s->spacemap, s->map_size); if (s->fd != -1) close(s->fd); if (s->mapfd != -1) @@ -294,8 +367,8 @@ static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary * av_log(h, AV_LOG_VERBOSE, "Opening cache file '%s' for URI: '%s'\n", s->cache_path, s->inner->filename); - s->fd = avpriv_open(s->cache_path, O_RDWR | O_CREAT, 0660); - s->mapfd = s->fd >= 0 ? avpriv_open(s->map_path, O_RDWR | O_CREAT, 0660) : -1; + s->fd = avpriv_open(s->cache_path, O_RDWR | O_CREAT | O_BINARY, 0660); + s->mapfd = s->fd >= 0 ? avpriv_open(s->map_path, O_RDWR | O_CREAT | O_BINARY, 0660) : -1; if (s->fd < 0 || s->mapfd < 0) { ret = AVERROR(errno); av_log(h, AV_LOG_ERROR, "Failed to open '%s': %s\n", @@ -334,7 +407,7 @@ static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary * if (ret < 0) goto fail; - /* If filesize is known, we can directly mmap() the cache file */ + /* If filesize is known, we can directly map the cache file */ ret = cache_map(h, filesize); if (ret < 0) { av_log(h, AV_LOG_WARNING, "Failed to map cache file: %s. Falling " @@ -367,31 +440,20 @@ static int cache_map(URLContext *h, int64_t filesize) return 0; if (s->cache_data) { - munmap(s->cache_data, s->cache_size); + av_file_unmap_shared(s->cache_data, s->cache_size); s->cache_data = NULL; s->cache_size = 0; } - struct stat st; - int ret = fstat(s->fd, &st); + /* The mapping extends the file to the file size; it can be shorter if + * another process wrote the correct filesize to the header but crashed + * right before actually successfully resizing the file. */ + void *map; + int ret = av_file_map_shared(s->fd, filesize, &map); if (ret < 0) - return AVERROR(errno); - - if (st.st_size != filesize) { - /* Ensure the file size is correct before mapping; this can happen if - * another process wrote the correct filesize to the header but - * crashed right before actually successfully resizing the file. */ - ret = ftruncate(s->fd, filesize); - if (ret < 0) - return AVERROR(errno); - } - - s->cache_data = mmap(NULL, filesize, PROT_READ | PROT_WRITE, MAP_SHARED, s->fd, 0); - if (s->cache_data == MAP_FAILED) { - s->cache_data = NULL; - return AVERROR(errno); - } + return ret; + s->cache_data = map; s->cache_size = filesize; return 0; } @@ -415,11 +477,9 @@ static int spacemap_remap(URLContext *h, size_t map_size) goto skip_resize; /* Lock the spacemap to ensure nobody else is currently resizing it */ - ret = flock(s->mapfd, LOCK_EX); - if (ret < 0) { - ret = AVERROR(errno); + ret = file_lock(s->mapfd); + if (ret < 0) goto fail; - } locked = 1; /* Refresh filesize after acquiring the lock */ @@ -432,28 +492,25 @@ static int spacemap_remap(URLContext *h, size_t map_size) if (st.st_size >= map_size) goto skip_resize; - ret = ftruncate(s->mapfd, map_size); - if (ret < 0) { - ret = AVERROR(errno); - goto fail; - } + /* The new mapping extends the file */ st.st_size = map_size; did_grow = 1; skip_resize: - if (s->spacemap) - munmap(s->spacemap, s->map_size); + av_file_unmap_shared(s->spacemap, s->map_size); + s->spacemap = NULL; s->map_size = st.st_size; - s->spacemap = mmap(NULL, s->map_size, PROT_READ | PROT_WRITE, MAP_SHARED, s->mapfd, 0); - if (s->spacemap == MAP_FAILED) { - s->spacemap = NULL; /* for munmap check */ + + void *map; + ret = av_file_map_shared(s->mapfd, s->map_size, &map); + if (ret < 0) { s->map_size = 0; - ret = AVERROR(errno); goto fail; } + s->spacemap = map; if (locked) { - flock(s->mapfd, LOCK_UN); + file_unlock(s->mapfd); locked = 0; } @@ -461,7 +518,7 @@ skip_resize: fail: if (locked) - flock(s->mapfd, LOCK_UN); + file_unlock(s->mapfd); av_log(h, AV_LOG_ERROR, "Failed to resize space map: %s\n", av_err2str(ret)); return ret; } @@ -485,7 +542,7 @@ static int spacemap_grow(URLContext *h, int64_t block) if (map_bytes < num_blocks) return AVERROR(EINVAL); /* overflow */ - const off_t old_size = s->map_size; + const size_t old_size = s->map_size; int ret = spacemap_remap(h, map_bytes); if (ret < 0) return ret; @@ -496,7 +553,7 @@ static int spacemap_grow(URLContext *h, int64_t block) av_log(h, AV_LOG_DEBUG, "%s %zu bytes, capacity: %"PRId64" blocks = %"PRId64" MB\n", ret ? "Resized spacemap to" : "Mapped spacemap with", - (size_t) s->map_size, num_blocks, + s->map_size, num_blocks, (num_blocks * (int64_t) s->block_size) >> 20); } return 0; @@ -557,7 +614,7 @@ static int spacemap_init(URLContext *h, const uint8_t hash[HASH_SIZE]) return ret; } -static int read_cache(SharedContext *s, uint8_t *buf, size_t size, off_t offset) +static int read_cache(SharedContext *s, uint8_t *buf, size_t size, int64_t offset) { if (s->cache_data) { av_assert1(offset + size <= s->cache_size); @@ -566,9 +623,9 @@ static int read_cache(SharedContext *s, uint8_t *buf, size_t size, off_t offset) } while (size) { - ssize_t ret = pread(s->fd, buf, size, offset); + int ret = file_pread(s->fd, buf, size, offset); if (ret <= 0) - return ret ? AVERROR(errno) : AVERROR_EOF; + return ret ? ret : AVERROR_EOF; buf += ret; offset += ret; size -= ret; @@ -577,7 +634,7 @@ static int read_cache(SharedContext *s, uint8_t *buf, size_t size, off_t offset) return 0; } -static int write_cache(SharedContext *s, const uint8_t *buf, size_t size, off_t offset) +static int write_cache(SharedContext *s, const uint8_t *buf, size_t size, int64_t offset) { if (s->cache_data) { av_assert1(offset + size <= s->cache_size); @@ -586,9 +643,9 @@ static int write_cache(SharedContext *s, const uint8_t *buf, size_t size, off_t } while (size) { - ssize_t ret = pwrite(s->fd, buf, size, offset); + int ret = file_pwrite(s->fd, buf, size, offset); if (ret <= 0) - return ret ? AVERROR(errno) : AVERROR(EIO); + return ret ? ret : AVERROR(EIO); buf += ret; offset += ret; size -= ret; -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
