[nginx] Mp4: of.directio_off flag to avoid unneeded syscalls.
details: http://freenginx.org/hg/nginx/rev/268e0fb3a16c
branches:
changeset: 9360:268e0fb3a16c
user: Maxim Dounin
date: Sun May 25 23:57:39 2025 +0300
description:
Mp4: of.directio_off flag to avoid unneeded syscalls.
When set, ngx_open_cached_file() won't try to enable directio on the file,
allowing the caller to do it itself, but will set is_directio flag in
ngx_open_file_info_t and in open file cache. The fact that directio needs
to be enabled by the caller is signalled in the of.is_directio_off output
flag.
This approach makes it possible to avoid unneeded syscalls when directio
is enabled, yet the caller needs it disabled for some initial file
processing, such as in the mp4 module.
diffstat:
src/core/ngx_open_file_cache.c | 8 +++-
src/core/ngx_open_file_cache.h | 2 ++
src/http/modules/ngx_http_mp4_module.c | 10 +-
3 files changed, 14 insertions(+), 6 deletions(-)
diffs (105 lines):
diff --git a/src/core/ngx_open_file_cache.c b/src/core/ngx_open_file_cache.c
--- a/src/core/ngx_open_file_cache.c
+++ b/src/core/ngx_open_file_cache.c
@@ -249,6 +249,7 @@ ngx_open_cached_file(ngx_open_file_cache
of->is_link = file->is_link;
of->is_exec = file->is_exec;
of->is_directio = file->is_directio;
+of->is_directio_off = 0;
if (!file->is_dir) {
file->count++;
@@ -313,6 +314,7 @@ ngx_open_cached_file(ngx_open_file_cache
}
of->is_directio = file->is_directio;
+of->is_directio_off = 0;
goto update;
}
@@ -920,7 +922,11 @@ ngx_open_and_stat_file(ngx_str_t *name,
}
if (of->directio <= ngx_file_size(&fi)) {
-if (ngx_directio_on(fd) == NGX_FILE_ERROR) {
+if (of->directio_off) {
+of->is_directio = 1;
+of->is_directio_off = 1;
+
+} else if (ngx_directio_on(fd) == NGX_FILE_ERROR) {
ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
ngx_directio_on_n " \"%V\" failed", name);
diff --git a/src/core/ngx_open_file_cache.h b/src/core/ngx_open_file_cache.h
--- a/src/core/ngx_open_file_cache.h
+++ b/src/core/ngx_open_file_cache.h
@@ -42,12 +42,14 @@ typedef struct {
unsigned log:1;
unsigned errors:1;
unsigned events:1;
+unsigned directio_off:1;
unsigned is_dir:1;
unsigned is_file:1;
unsigned is_link:1;
unsigned is_exec:1;
unsigned is_directio:1;
+unsigned is_directio_off:1;
} ngx_open_file_info_t;
diff --git a/src/http/modules/ngx_http_mp4_module.c
b/src/http/modules/ngx_http_mp4_module.c
--- a/src/http/modules/ngx_http_mp4_module.c
+++ b/src/http/modules/ngx_http_mp4_module.c
@@ -479,7 +479,7 @@ ngx_http_mp4_handler(ngx_http_request_t
u_char*last;
size_t root;
ngx_int_t rc, start, end;
-ngx_uint_t level, length, directio;
+ngx_uint_t level, length;
ngx_str_t path, value;
ngx_log_t *log;
ngx_buf_t *b;
@@ -520,6 +520,7 @@ ngx_http_mp4_handler(ngx_http_request_t
of.read_ahead = clcf->read_ahead;
of.directio = clcf->directio;
+of.directio_off = 1;
of.valid = clcf->open_file_cache_valid;
of.min_uses = clcf->open_file_cache_min_uses;
of.errors = clcf->open_file_cache_errors;
@@ -579,7 +580,6 @@ ngx_http_mp4_handler(ngx_http_request_t
start = -1;
length = 0;
-directio = 0;
r->headers_out.content_length_n = of.size;
mp4 = NULL;
b = NULL;
@@ -615,7 +615,7 @@ ngx_http_mp4_handler(ngx_http_request_t
if (start >= 0) {
r->single_range = 1;
-if (of.is_directio) {
+if (of.is_directio && !of.is_directio_off) {
/*
* DIRECTIO is set on transfer only
@@ -627,7 +627,7 @@ ngx_http_mp4_handler(ngx_http_request_t
ngx_directio_off_n " \"%s\" failed", path.data);
}
-directio = 1;
+of.is_directio_off = 1;
}
mp4 = ngx_pcalloc(r->pool, sizeof(ngx_http_mp4_file_t));
@@ -673,7 +673,7 @@ ngx_http_mp4_handler(ngx_http_request_t
log->action = "sending mp4 to client";
-if (directio) {
+if (of.is_directio_off) {
/* DIRECTIO was switched off, restore it */
[nginx] Mp4: fixed directio usage with open_file_cache.
details: http://freenginx.org/hg/nginx/rev/314192d67a91
branches:
changeset: 9359:314192d67a91
user: Maxim Dounin
date: Sun May 25 23:57:28 2025 +0300
description:
Mp4: fixed directio usage with open_file_cache.
With open_file_cache, if directio is enabled on a file descriptor after
opening the file, other consumers won't know about directio being enabled
and will use unaligned reads, leading to EINVAL errors from pread()
on Linux.
Further, if a file descriptor with directio enabled will be returned to
the mp4 module itself, during mp4 file processing it will be used in
assumption that directio is not enabled.
Fix is to use of.directio and ngx_open_cached_file() to enable directio,
and switch it off during mp4 file processing.
While this does some unneeded syscalls if the file is actually just opened
and we have to parse the file, this shouldn't be significant compared to
other mp4 file processing costs.
diffstat:
src/http/modules/ngx_http_mp4_module.c | 36 -
1 files changed, 22 insertions(+), 14 deletions(-)
diffs (86 lines):
diff --git a/src/http/modules/ngx_http_mp4_module.c
b/src/http/modules/ngx_http_mp4_module.c
--- a/src/http/modules/ngx_http_mp4_module.c
+++ b/src/http/modules/ngx_http_mp4_module.c
@@ -479,7 +479,7 @@ ngx_http_mp4_handler(ngx_http_request_t
u_char*last;
size_t root;
ngx_int_t rc, start, end;
-ngx_uint_t level, length;
+ngx_uint_t level, length, directio;
ngx_str_t path, value;
ngx_log_t *log;
ngx_buf_t *b;
@@ -519,7 +519,7 @@ ngx_http_mp4_handler(ngx_http_request_t
ngx_memzero(&of, sizeof(ngx_open_file_info_t));
of.read_ahead = clcf->read_ahead;
-of.directio = NGX_MAX_OFF_T_VALUE;
+of.directio = clcf->directio;
of.valid = clcf->open_file_cache_valid;
of.min_uses = clcf->open_file_cache_min_uses;
of.errors = clcf->open_file_cache_errors;
@@ -579,6 +579,7 @@ ngx_http_mp4_handler(ngx_http_request_t
start = -1;
length = 0;
+directio = 0;
r->headers_out.content_length_n = of.size;
mp4 = NULL;
b = NULL;
@@ -614,6 +615,21 @@ ngx_http_mp4_handler(ngx_http_request_t
if (start >= 0) {
r->single_range = 1;
+if (of.is_directio) {
+
+/*
+ * DIRECTIO is set on transfer only
+ * to allow kernel to cache "moov" atom
+ */
+
+if (ngx_directio_off(of.fd) == NGX_FILE_ERROR) {
+ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
+ ngx_directio_off_n " \"%s\" failed", path.data);
+}
+
+directio = 1;
+}
+
mp4 = ngx_pcalloc(r->pool, sizeof(ngx_http_mp4_file_t));
if (mp4 == NULL) {
return NGX_HTTP_INTERNAL_SERVER_ERROR;
@@ -622,6 +638,7 @@ ngx_http_mp4_handler(ngx_http_request_t
mp4->file.fd = of.fd;
mp4->file.name = path;
mp4->file.log = r->connection->log;
+mp4->file.directio = of.is_directio;
mp4->end = of.size;
mp4->start = (ngx_uint_t) start;
mp4->length = length;
@@ -656,23 +673,14 @@ ngx_http_mp4_handler(ngx_http_request_t
log->action = "sending mp4 to client";
-if (clcf->directio <= of.size) {
-
-/*
- * DIRECTIO is set on transfer only
- * to allow kernel to cache "moov" atom
- */
+if (directio) {
+
+/* DIRECTIO was switched off, restore it */
if (ngx_directio_on(of.fd) == NGX_FILE_ERROR) {
ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
ngx_directio_on_n " \"%s\" failed", path.data);
}
-
-of.is_directio = 1;
-
-if (mp4) {
-mp4->file.directio = 1;
-}
}
r->headers_out.status = NGX_HTTP_OK;
[nginx-tests] Tests: mp4 tests with directio and open_file_cache.
details: http://freenginx.org/hg/nginx-tests/rev/ba4f76b3b823
branches:
changeset: 2008:ba4f76b3b823
user: Maxim Dounin
date: Mon May 26 00:02:51 2025 +0300
description:
Tests: mp4 tests with directio and open_file_cache.
diffstat:
mp4_directio.t | 78 ++
1 files changed, 78 insertions(+), 0 deletions(-)
diffs (83 lines):
diff --git a/mp4_directio.t b/mp4_directio.t
new file mode 100644
--- /dev/null
+++ b/mp4_directio.t
@@ -0,0 +1,78 @@
+#!/usr/bin/perl
+
+# (C) Maxim Dounin
+
+# Test for directio support in mp4 module.
+
+###
+
+use warnings;
+use strict;
+
+use Test::More;
+
+BEGIN { use FindBin; chdir($FindBin::Bin); }
+
+use lib 'lib';
+use Test::Nginx;
+
+###
+
+select STDERR; $| = 1;
+select STDOUT; $| = 1;
+
+my $t = Test::Nginx->new()->has(qw/http mp4/)->has_daemon('ffmpeg')
+ ->write_file_expand('nginx.conf', <<'EOF');
+
+%%TEST_GLOBALS%%
+
+daemon off;
+
+events {
+}
+
+http {
+%%TEST_GLOBALS_HTTP%%
+
+server {
+listen 127.0.0.1:8080;
+server_name localhost;
+
+location / {
+mp4;
+open_file_cache max=10;
+directio 0;
+}
+}
+}
+
+EOF
+
+plan(skip_all => 'no lavfi')
+ unless grep /lavfi/, `ffmpeg -loglevel quiet -formats`;
+plan(skip_all => 'no libx264 or libopenh264')
+ unless grep /libx264|libopenh264/, `ffmpeg -loglevel quiet -encoders`;
+system('ffmpeg -nostdin -loglevel quiet -y '
+ . '-f lavfi -i testsrc=duration=10:size=320x200:rate=15 '
+ . '-g 15 -c:v h264 '
+ . "${\($t->testdir())}/test.mp4") == 0
+ or die "Can't create mp4 file: $!";
+
+$t->run()->plan(2);
+
+###
+
+# mp4 module uses unaligned reads while parsing mp4 file, though
+# failed to disable directio if a file with directio enabled was
+# returned from open file cache
+
+like(http_get('/test.mp4?start=1.0'), qr/ 200 /, 'mp4 directio first');
+
+TODO: {
+local $TODO = 'not yet' unless $t->has_version('1.29.0') or $^O ne 'linux';
+
+like(http_get('/test.mp4?start=1.0'), qr/ 200 /, 'mp4 directio cached');
+
+}
+
+###
[nginx] Cache: directio support when reading cache files.
details: http://freenginx.org/hg/nginx/rev/e2a6fefd81db
branches:
changeset: 9361:e2a6fefd81db
user: Maxim Dounin
date: Mon May 26 00:04:45 2025 +0300
description:
Cache: directio support when reading cache files.
diffstat:
src/http/ngx_http_file_cache.c | 32
1 files changed, 28 insertions(+), 4 deletions(-)
diffs (60 lines):
diff --git a/src/http/ngx_http_file_cache.c b/src/http/ngx_http_file_cache.c
--- a/src/http/ngx_http_file_cache.c
+++ b/src/http/ngx_http_file_cache.c
@@ -353,7 +353,7 @@ ngx_http_file_cache_open(ngx_http_reques
of.valid = clcf->open_file_cache_valid;
of.min_uses = clcf->open_file_cache_min_uses;
of.events = clcf->open_file_cache_events;
-of.directio = NGX_OPEN_FILE_DIRECTIO_OFF;
+of.directio = clcf->directio;
of.read_ahead = clcf->read_ahead;
if (ngx_open_cached_file(clcf->open_file_cache, &c->file.name, &of,
r->pool)
@@ -380,13 +380,36 @@ ngx_http_file_cache_open(ngx_http_reques
c->file.fd = of.fd;
c->file.log = r->connection->log;
+c->file.directio = of.is_directio;
c->uniq = of.uniq;
c->length = of.size;
c->fs_size = (of.fs_size + cache->bsize - 1) / cache->bsize;
-c->buf = ngx_create_temp_buf(r->pool, c->body_start);
-if (c->buf == NULL) {
-return NGX_ERROR;
+if (of.is_directio) {
+
+c->body_start = ngx_align(c->body_start, clcf->directio_alignment);
+
+c->buf = ngx_calloc_buf(r->pool);
+if (c->buf == NULL) {
+return NGX_ERROR;
+}
+
+c->buf->start = ngx_pmemalign(r->pool, c->body_start,
+ clcf->directio_alignment);
+if (c->buf->start == NULL) {
+return NGX_ERROR;
+}
+
+c->buf->pos = c->buf->start;
+c->buf->last = c->buf->start;
+c->buf->end = c->buf->start + c->body_start;
+c->buf->temporary = 1;
+
+} else {
+c->buf = ngx_create_temp_buf(r->pool, c->body_start);
+if (c->buf == NULL) {
+return NGX_ERROR;
+}
}
return ngx_http_file_cache_read(r, c);
@@ -1663,6 +1686,7 @@ ngx_http_cache_send(ngx_http_request_t *
b->file->fd = c->file.fd;
b->file->name = c->file.name;
b->file->log = r->connection->log;
+b->file->directio = c->file.directio;
out.buf = b;
out.next = NULL;
[PATCH 0 of 5] open file cache fixes
Hello! The following patch series implements some open file cache fixes. Notably, directio handling is fixed to be compatible with threaded IO. -- Maxim Dounin
[PATCH 1 of 5] Disabled open_file_cache on platforms without pread()
# HG changeset patch
# User Maxim Dounin
# Date 1748208420 -10800
# Mon May 26 00:27:00 2025 +0300
# Node ID 811bf7dc0d46e5e6d69e700a1a5a3852d4ba0381
# Parent e2a6fefd81dbe201a88876a95f34620df87259a2
Disabled open_file_cache on platforms without pread().
Current open file cache code cannot properly work on platforms without
pread(), since file->sys_offset is not shared across files. Further, it
is not set on file initialization after ngx_open_cached_file(), leading
to incorrect value 0 instead of non-zero current offset for cached file
descriptors.
Since platforms without pread() are rather exotic nowadays, fix is to
disable open_file_cache for them.
diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c
--- a/src/http/ngx_http_core_module.c
+++ b/src/http/ngx_http_core_module.c
@@ -4981,6 +4981,8 @@ ngx_http_core_error_page(ngx_conf_t *cf,
static char *
ngx_http_core_open_file_cache(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
{
+#if (NGX_HAVE_PREAD || NGX_WIN32)
+
ngx_http_core_loc_conf_t *clcf = conf;
time_t inactive;
@@ -5048,11 +5050,17 @@ ngx_http_core_open_file_cache(ngx_conf_t
}
clcf->open_file_cache = ngx_open_file_cache_init(cf->pool, max, inactive);
-if (clcf->open_file_cache) {
-return NGX_CONF_OK;
-}
-
-return NGX_CONF_ERROR;
+if (clcf->open_file_cache == NULL) {
+return NGX_CONF_ERROR;
+}
+
+#else
+ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
+ "\"open_file_cache\" is not supported "
+ "on this platform, ignored");
+#endif
+
+return NGX_CONF_OK;
}
[PATCH 2 of 5] Open file cache: style
# HG changeset patch
# User Maxim Dounin
# Date 1748208424 -10800
# Mon May 26 00:27:04 2025 +0300
# Node ID 8cc90a3e3e74d31a92b2732f6f4003684a011fcd
# Parent 811bf7dc0d46e5e6d69e700a1a5a3852d4ba0381
Open file cache: style.
diff --git a/src/core/ngx_open_file_cache.c b/src/core/ngx_open_file_cache.c
--- a/src/core/ngx_open_file_cache.c
+++ b/src/core/ngx_open_file_cache.c
@@ -974,7 +974,7 @@ ngx_open_file_add_event(ngx_open_file_ca
file->use_event = 0;
file->event = ngx_calloc(sizeof(ngx_event_t), log);
-if (file->event== NULL) {
+if (file->event == NULL) {
return;
}
diff --git a/src/core/ngx_open_file_cache.h b/src/core/ngx_open_file_cache.h
--- a/src/core/ngx_open_file_cache.h
+++ b/src/core/ngx_open_file_cache.h
@@ -53,9 +53,7 @@ typedef struct {
} ngx_open_file_info_t;
-typedef struct ngx_cached_open_file_s ngx_cached_open_file_t;
-
-struct ngx_cached_open_file_s {
+typedef struct {
ngx_rbtree_node_tnode;
ngx_queue_t queue;
@@ -87,7 +85,7 @@ struct ngx_cached_open_file_s {
unsigned is_directio:1;
ngx_event_t *event;
-};
+} ngx_cached_open_file_t;
typedef struct {
[PATCH 3 of 5] Open file cache: disable_symlinks with open_file_cache_events
# HG changeset patch
# User Maxim Dounin
# Date 1748208427 -10800
# Mon May 26 00:27:07 2025 +0300
# Node ID f84080a83901176c0dd95681b19290f070de045c
# Parent 8cc90a3e3e74d31a92b2732f6f4003684a011fcd
Open file cache: disable_symlinks with open_file_cache_events.
Previously, different disable_symlinks settings were respected when
retrieving a file from open file cache when using periodic retest, but
were ignored with using open_file_cache_events.
Fix is to test disable_symlinks settings in all cases, and retest the
file if requested settings are different from what we already have in
the cache.
diff --git a/src/core/ngx_open_file_cache.c b/src/core/ngx_open_file_cache.c
--- a/src/core/ngx_open_file_cache.c
+++ b/src/core/ngx_open_file_cache.c
@@ -227,15 +227,15 @@ ngx_open_cached_file(ngx_open_file_cache
goto add_event;
}
-if (file->use_event
-|| (file->event == NULL
-&& (of->uniq == 0 || of->uniq == file->uniq)
-&& now - file->created < of->valid
+if ((file->use_event
+ || (file->event == NULL
+ && (of->uniq == 0 || of->uniq == file->uniq)
+ && now - file->created < of->valid))
#if (NGX_HAVE_OPENAT)
-&& of->disable_symlinks == file->disable_symlinks
-&& of->disable_symlinks_from == file->disable_symlinks_from
+&& of->disable_symlinks == file->disable_symlinks
+&& of->disable_symlinks_from == file->disable_symlinks_from
#endif
-))
+)
{
if (file->err == 0) {
[PATCH 4 of 5] Open file cache: fixed file->uses loss on retest
# HG changeset patch
# User Maxim Dounin
# Date 1748208430 -10800
# Mon May 26 00:27:10 2025 +0300
# Node ID 6032949667f1e5fda9ce364d6eb1668074274c56
# Parent f84080a83901176c0dd95681b19290f070de045c
Open file cache: fixed file->uses loss on retest.
If an open file was reopened during a retest, but was in use by another
request, the cache entry was re-created with file->uses set to 1. Fix
is to preserve existing file->uses.
diff --git a/src/core/ngx_open_file_cache.c b/src/core/ngx_open_file_cache.c
--- a/src/core/ngx_open_file_cache.c
+++ b/src/core/ngx_open_file_cache.c
@@ -147,6 +147,7 @@ ngx_open_cached_file(ngx_open_file_cache
time_t now;
uint32_thash;
ngx_int_t rc;
+ngx_uint_t uses;
ngx_file_info_t fi;
ngx_pool_cleanup_t *cln;
ngx_cached_open_file_t *file;
@@ -348,6 +349,8 @@ ngx_open_cached_file(ngx_open_file_cache
file->close = 1;
+uses = file->uses;
+
goto create;
}
@@ -359,6 +362,8 @@ ngx_open_cached_file(ngx_open_file_cache
goto failed;
}
+uses = 1;
+
create:
if (cache->current >= cache->max) {
@@ -387,7 +392,7 @@ create:
cache->current++;
-file->uses = 1;
+file->uses = uses;
file->count = 0;
file->use_event = 0;
file->event = NULL;
[PATCH 5 of 5] Open file cached: correct directio handling with threads
# HG changeset patch
# User Maxim Dounin
# Date 1748208437 -10800
# Mon May 26 00:27:17 2025 +0300
# Node ID e3e07705481df3c60a3bac8af9101323ee7ceb83
# Parent 6032949667f1e5fda9ce364d6eb1668074274c56
Open file cached: correct directio handling with threads.
On Linux with open_file_cache and directio enabled along with threaded IO,
if a file descriptor is shared among multiple requests, one request might
unexpectedly switch directio on while another request does an unaligned
read in a thread, leading to EINVAL from pread().
To fix this, now all directio changes are done with reference counting
when open file cache is used, and directio is only re-enabled on a file
descriptor when there are no outstanding requests to switch it off.
diff --git a/src/core/ngx_open_file_cache.c b/src/core/ngx_open_file_cache.c
--- a/src/core/ngx_open_file_cache.c
+++ b/src/core/ngx_open_file_cache.c
@@ -41,7 +41,8 @@ static void ngx_open_file_add_event(ngx_
ngx_cached_open_file_t *file, ngx_open_file_info_t *of, ngx_log_t *log);
static void ngx_open_file_cleanup(void *data);
static void ngx_close_cached_file(ngx_open_file_cache_t *cache,
-ngx_cached_open_file_t *file, ngx_uint_t min_uses, ngx_log_t *log);
+ngx_cached_open_file_t *file, ngx_uint_t min_uses, ngx_uint_t directio_off,
+ngx_log_t *log);
static void ngx_open_file_del_event(ngx_cached_open_file_t *file);
static void ngx_expire_old_cached_files(ngx_open_file_cache_t *cache,
ngx_uint_t n, ngx_log_t *log);
@@ -51,6 +52,8 @@ static ngx_cached_open_file_t *
ngx_open_file_lookup(ngx_open_file_cache_t *cache, ngx_str_t *name,
uint32_t hash);
static void ngx_open_file_cache_remove(ngx_event_t *ev);
+static ngx_open_file_cache_cleanup_t *
+ngx_open_file_cache_get_cleanup(ngx_pool_t *p, ngx_fd_t fd);
ngx_open_file_cache_t *
@@ -118,7 +121,7 @@ ngx_open_file_cache_cleanup(void *data)
if (!file->err && !file->is_dir) {
file->close = 1;
file->count = 0;
-ngx_close_cached_file(cache, file, 0, ngx_cycle->log);
+ngx_close_cached_file(cache, file, 0, 0, ngx_cycle->log);
} else {
ngx_free(file->name);
@@ -396,6 +399,7 @@ create:
file->count = 0;
file->use_event = 0;
file->event = NULL;
+file->directio_off = of->is_directio_off;
add_event:
@@ -449,6 +453,7 @@ found:
ofcln->cache = cache;
ofcln->file = file;
ofcln->min_uses = of->min_uses;
+ofcln->directio_off = of->is_directio_off;
ofcln->log = pool->log;
}
@@ -1032,7 +1037,8 @@ ngx_open_file_cleanup(void *data)
c->file->count--;
-ngx_close_cached_file(c->cache, c->file, c->min_uses, c->log);
+ngx_close_cached_file(c->cache, c->file, c->min_uses, c->directio_off,
+ c->log);
/* drop one or two expired open files */
ngx_expire_old_cached_files(c->cache, 1, c->log);
@@ -1041,12 +1047,25 @@ ngx_open_file_cleanup(void *data)
static void
ngx_close_cached_file(ngx_open_file_cache_t *cache,
-ngx_cached_open_file_t *file, ngx_uint_t min_uses, ngx_log_t *log)
+ngx_cached_open_file_t *file, ngx_uint_t min_uses, ngx_uint_t directio_off,
+ngx_log_t *log)
{
ngx_log_debug5(NGX_LOG_DEBUG_CORE, log, 0,
"close cached open file: %s, fd:%d, c:%d, u:%d, %d",
file->name, file->fd, file->count, file->uses, file->close);
+if (directio_off) {
+file->directio_off--;
+
+if (file->directio_off == 0) {
+if (ngx_directio_on(file->fd) == NGX_FILE_ERROR) {
+ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
+ ngx_directio_on_n " \"%s\" failed",
+ file->name);
+}
+}
+}
+
if (!file->close) {
file->accessed = ngx_time();
@@ -1143,7 +1162,7 @@ ngx_expire_old_cached_files(ngx_open_fil
if (!file->err && !file->is_dir) {
file->close = 1;
-ngx_close_cached_file(cache, file, 0, log);
+ngx_close_cached_file(cache, file, 0, 0, log);
} else {
ngx_free(file->name);
@@ -1255,10 +1274,92 @@ ngx_open_file_cache_remove(ngx_event_t *
file->close = 1;
-ngx_close_cached_file(fev->cache, file, 0, ev->log);
+ngx_close_cached_file(fev->cache, file, 0, 0, ev->log);
/* free memory only when fev->cache and fev->file are already not needed */
ngx_free(ev->data);
ngx_free(ev);
}
+
+
+ngx_int_t
+ngx_open_file_directio_on(ngx_fd_t fd, ngx_pool_t *pool)
+{
+ngx_open_file_cache_cleanup_t *c;
+
+/*
+ * DIRECTIO is only re-enabled on a file descriptor
+ * when there are no outstanding requests to switch it off
+ */
+
+c = ngx_open_file_cache_get_cleanup(pool, fd);
+
+if (c) {
+if (!c->directio_off) {
+return NGX_OK;
+}
+
+c->directio_off = 0;
+
[PATCH 3 of 6] Cleaned up iov_base, msg_name, msg_control type casts
# HG changeset patch
# User Maxim Dounin
# Date 1748209475 -10800
# Mon May 26 00:44:35 2025 +0300
# Node ID 4eed7851024b4ca5d77107676025e84324e23c74
# Parent 38f1f374523ff2630e59f6a722fd6f9d7f70ac01
Cleaned up iov_base, msg_name, msg_control type casts.
In old OSes iov_base, msg_name, and msg_control fields were caddr_t
("char *" in case of iov_base), though later were changed to "void *".
In particular, in FreeBSD this change happened around FreeBSD 4.0.
With newer "void *" type no type casts are needed, but with caddr_t
type casts were required in most cases.
Previously, casts were used inconsistently across the code: to caddr_t,
to "void *", or no casts at all in some recent code. With this change,
casts to "void *" are consistently used in all relevant places.
diff --git a/src/event/ngx_event_udp.c b/src/event/ngx_event_udp.c
--- a/src/event/ngx_event_udp.c
+++ b/src/event/ngx_event_udp.c
@@ -70,14 +70,14 @@ ngx_event_recvmsg(ngx_event_t *ev)
iov[0].iov_base = (void *) buffer;
iov[0].iov_len = sizeof(buffer);
-msg.msg_name = &sa;
+msg.msg_name = (void *) &sa;
msg.msg_namelen = sizeof(ngx_sockaddr_t);
msg.msg_iov = iov;
msg.msg_iovlen = 1;
#if (NGX_HAVE_ADDRINFO_CMSG)
if (ls->wildcard) {
-msg.msg_control = &msg_control;
+msg.msg_control = (void *) &msg_control;
msg.msg_controllen = sizeof(msg_control);
ngx_memzero(&msg_control, sizeof(msg_control));
@@ -108,7 +108,7 @@ ngx_event_recvmsg(ngx_event_t *ev)
}
#endif
-sockaddr = msg.msg_name;
+sockaddr = (void *) msg.msg_name;
socklen = msg.msg_namelen;
if (socklen > (socklen_t) sizeof(ngx_sockaddr_t)) {
diff --git a/src/event/quic/ngx_event_quic_output.c
b/src/event/quic/ngx_event_quic_output.c
--- a/src/event/quic/ngx_event_quic_output.c
+++ b/src/event/quic/ngx_event_quic_output.c
@@ -410,16 +410,16 @@ ngx_quic_send_segments(ngx_connection_t
ngx_memzero(&msg, sizeof(struct msghdr));
ngx_memzero(msg_control, sizeof(msg_control));
+iov.iov_base = (void *) buf;
iov.iov_len = len;
-iov.iov_base = buf;
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
-msg.msg_name = sockaddr;
+msg.msg_name = (void *) sockaddr;
msg.msg_namelen = socklen;
-msg.msg_control = msg_control;
+msg.msg_control = (void *) msg_control;
msg.msg_controllen = sizeof(msg_control);
cmsg = CMSG_FIRSTHDR(&msg);
@@ -698,19 +698,19 @@ ngx_quic_send(ngx_connection_t *c, u_cha
ngx_memzero(&msg, sizeof(struct msghdr));
+iov.iov_base = (void *) buf;
iov.iov_len = len;
-iov.iov_base = buf;
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
-msg.msg_name = sockaddr;
+msg.msg_name = (void *) sockaddr;
msg.msg_namelen = socklen;
#if (NGX_HAVE_ADDRINFO_CMSG)
if (c->listening && c->listening->wildcard && c->local_sockaddr) {
-msg.msg_control = msg_control;
+msg.msg_control = (void *) msg_control;
msg.msg_controllen = sizeof(msg_control);
ngx_memzero(msg_control, sizeof(msg_control));
diff --git a/src/event/quic/ngx_event_quic_udp.c
b/src/event/quic/ngx_event_quic_udp.c
--- a/src/event/quic/ngx_event_quic_udp.c
+++ b/src/event/quic/ngx_event_quic_udp.c
@@ -68,14 +68,14 @@ ngx_quic_recvmsg(ngx_event_t *ev)
iov[0].iov_base = (void *) buffer;
iov[0].iov_len = sizeof(buffer);
-msg.msg_name = &sa;
+msg.msg_name = (void *) &sa;
msg.msg_namelen = sizeof(ngx_sockaddr_t);
msg.msg_iov = iov;
msg.msg_iovlen = 1;
#if (NGX_HAVE_ADDRINFO_CMSG)
if (ls->wildcard) {
-msg.msg_control = &msg_control;
+msg.msg_control = (void *) &msg_control;
msg.msg_controllen = sizeof(msg_control);
ngx_memzero(&msg_control, sizeof(msg_control));
@@ -106,7 +106,7 @@ ngx_quic_recvmsg(ngx_event_t *ev)
}
#endif
-sockaddr = msg.msg_name;
+sockaddr = (void *) msg.msg_name;
socklen = msg.msg_namelen;
if (socklen > (socklen_t) sizeof(ngx_sockaddr_t)) {
diff --git a/src/os/unix/ngx_channel.c b/src/os/unix/ngx_channel.c
--- a/src/os/unix/ngx_channel.c
+++ b/src/os/unix/ngx_channel.c
@@ -31,7 +31,7 @@ ngx_write_channel(ngx_socket_t s, ngx_ch
msg.msg_controllen = 0;
} else {
-msg.msg_control = (caddr_t) &cmsg;
+msg.msg_control = (void *) &cmsg;
msg.msg_controllen = sizeof(cmsg);
ngx_memzero(&cmsg, sizeof(cmsg));
@@ -68,7 +68,7 @@ ngx_write_channel(ngx_socket_t s, ngx_ch
#endif
-iov[0].iov_base = (char *) ch;
+iov[0].iov_base = (void *) ch;
iov[0].iov_len = size;
msg.msg_name = NULL;
@@ -109,7 +109,7 @@ ngx_read_channel(ngx_socket_t s, ngx_cha
int fd;
#endif
-iov[0].iov_base = (char *) ch;
+iov[0].iov_base = (void *) ch;
iov[0].iov_len =
[PATCH 5 of 6] Fixed unused variable in ngx_set_srcaddr_cmsg() on MINIX
# HG changeset patch
# User Maxim Dounin
# Date 1748209507 -10800
# Mon May 26 00:45:07 2025 +0300
# Node ID 7a595fa6ade10f801f6e9bc84a7fd8fc1f9cca21
# Parent 3e11162ed39312387fc79a731071b0a50aa15acc
Fixed unused variable in ngx_set_srcaddr_cmsg() on MINIX.
diff --git a/src/os/unix/ngx_udp_sendmsg_chain.c
b/src/os/unix/ngx_udp_sendmsg_chain.c
--- a/src/os/unix/ngx_udp_sendmsg_chain.c
+++ b/src/os/unix/ngx_udp_sendmsg_chain.c
@@ -245,7 +245,10 @@ ngx_sendmsg_vec(ngx_connection_t *c, ngx
size_t
ngx_set_srcaddr_cmsg(struct cmsghdr *cmsg, struct sockaddr *local_sockaddr)
{
+#if (NGX_HAVE_IP_SENDSRCADDR || NGX_HAVE_IP_PKTINFO \
+ || (NGX_HAVE_INET6 && NGX_HAVE_IPV6_RECVPKTINFO))
size_tlen;
+#endif
#if (NGX_HAVE_IP_SENDSRCADDR)
struct in_addr *addr;
struct sockaddr_in *sin;
[PATCH 1 of 6] Configure: fixed nobody/nogroup detection stderr redirection
# HG changeset patch
# User Maxim Dounin
# Date 1748209461 -10800
# Mon May 26 00:44:21 2025 +0300
# Node ID 03f5c84e7e8c05ce16f6f36452e4cb818c85dde6
# Parent e2a6fefd81dbe201a88876a95f34620df87259a2
Configure: fixed nobody/nogroup detection stderr redirection.
Previously, "grep: /etc/group: No such file or directory" errors were
printed on systems without the /etc/group file along with the configure
output. Fix is to use correct stderr redirection.
diff --git a/auto/unix b/auto/unix
--- a/auto/unix
+++ b/auto/unix
@@ -7,13 +7,13 @@ NGX_USER=${NGX_USER:-nobody}
if [ -z "$NGX_GROUP" ]; then
if [ $NGX_USER = nobody ]; then
-if grep nobody /etc/group 2>&1 >/dev/null; then
+if grep nobody /etc/group >/dev/null 2>&1; then
echo "checking for nobody group ... found"
NGX_GROUP=nobody
else
echo "checking for nobody group ... not found"
-if grep nogroup /etc/group 2>&1 >/dev/null; then
+if grep nogroup /etc/group >/dev/null 2>&1; then
echo "checking for nogroup group ... found"
NGX_GROUP=nogroup
else
[PATCH 6 of 6] Compatibility with systems without SA_SIGINFO in sigaction()
# HG changeset patch
# User Maxim Dounin
# Date 1748209526 -10800
# Mon May 26 00:45:26 2025 +0300
# Node ID 95e5cc6c73d1b94a59c4eea26bd2956a1cdc2f3d
# Parent 7a595fa6ade10f801f6e9bc84a7fd8fc1f9cca21
Compatibility with systems without SA_SIGINFO in sigaction().
This change restores compatibility with some older systems without
SA_SIGINFO support, broken by 6985:23ecffd5bcfe. Notably, this fixes
compilation on MINIX.
diff --git a/auto/unix b/auto/unix
--- a/auto/unix
+++ b/auto/unix
@@ -1043,3 +1043,16 @@ ngx_feature_test='struct addrinfo *res;
if (getaddrinfo("localhost", NULL, NULL, &res) != 0) return
1;
freeaddrinfo(res)'
. auto/feature
+
+
+ngx_feature="sigaction(SA_SIGINFO)"
+ngx_feature_name="NGX_HAVE_SIGINFO"
+ngx_feature_run=no
+ngx_feature_incs="#include "
+ngx_feature_path=
+ngx_feature_libs=
+ngx_feature_test="struct sigaction sa;
+ sa.sa_flags = SA_SIGINFO;
+ sa.sa_sigaction = NULL;
+ if (sigaction(0, &sa, NULL) == -1) return 1;"
+. auto/feature
diff --git a/src/os/unix/ngx_process.c b/src/os/unix/ngx_process.c
--- a/src/os/unix/ngx_process.c
+++ b/src/os/unix/ngx_process.c
@@ -15,13 +15,21 @@ typedef struct {
int signo;
char *signame;
char *name;
+#if (NGX_HAVE_SIGINFO)
void (*handler)(int signo, siginfo_t *siginfo, void *ucontext);
+#else
+void (*handler)(int signo);
+#endif
} ngx_signal_t;
static void ngx_execute_proc(ngx_cycle_t *cycle, void *data);
+#if (NGX_HAVE_SIGINFO)
static void ngx_signal_handler(int signo, siginfo_t *siginfo, void *ucontext);
+#else
+static void ngx_signal_handler(int signo);
+#endif
static void ngx_process_get_status(void);
static void ngx_unlock_mutexes(ngx_pid_t pid);
@@ -291,8 +299,12 @@ ngx_init_signals(ngx_log_t *log)
ngx_memzero(&sa, sizeof(struct sigaction));
if (sig->handler) {
+#if (NGX_HAVE_SIGINFO)
sa.sa_sigaction = sig->handler;
sa.sa_flags = SA_SIGINFO;
+#else
+sa.sa_handler = sig->handler;
+#endif
} else {
sa.sa_handler = SIG_IGN;
@@ -315,8 +327,13 @@ ngx_init_signals(ngx_log_t *log)
}
+#if (NGX_HAVE_SIGINFO)
static void
ngx_signal_handler(int signo, siginfo_t *siginfo, void *ucontext)
+#else
+static void
+ngx_signal_handler(int signo)
+#endif
{
char*action;
ngx_int_tignore;
@@ -441,12 +458,15 @@ ngx_signal_handler(int signo, siginfo_t
break;
}
+#if (NGX_HAVE_SIGINFO)
if (siginfo && siginfo->si_pid) {
ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0,
"signal %d (%s) received from %P%s",
signo, sig->signame, siginfo->si_pid, action);
-} else {
+} else
+#endif
+{
ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0,
"signal %d (%s) received%s",
signo, sig->signame, action);
[PATCH 4 of 6] Adjusted mmap() type casts
# HG changeset patch
# User Maxim Dounin
# Date 1748209484 -10800
# Mon May 26 00:44:44 2025 +0300
# Node ID 3e11162ed39312387fc79a731071b0a50aa15acc
# Parent 4eed7851024b4ca5d77107676025e84324e23c74
Adjusted mmap() type casts.
Type casting mmap() result explicitly to (u_char *) is only needed on
systems where mmap() returns caddr_t, but on these systems MAP_FAILED
also needs to be casted. Added appropriate type casts.
diff --git a/src/os/unix/ngx_shmem.c b/src/os/unix/ngx_shmem.c
--- a/src/os/unix/ngx_shmem.c
+++ b/src/os/unix/ngx_shmem.c
@@ -18,7 +18,7 @@ ngx_shm_alloc(ngx_shm_t *shm)
PROT_READ|PROT_WRITE,
MAP_ANON|MAP_SHARED, -1, 0);
-if (shm->addr == MAP_FAILED) {
+if (shm->addr == (u_char *) MAP_FAILED) {
ngx_log_error(NGX_LOG_ALERT, shm->log, ngx_errno,
"mmap(MAP_ANON|MAP_SHARED, %uz) failed", shm->size);
return NGX_ERROR;
@@ -55,7 +55,7 @@ ngx_shm_alloc(ngx_shm_t *shm)
shm->addr = (u_char *) mmap(NULL, shm->size, PROT_READ|PROT_WRITE,
MAP_SHARED, fd, 0);
-if (shm->addr == MAP_FAILED) {
+if (shm->addr == (u_char *) MAP_FAILED) {
ngx_log_error(NGX_LOG_ALERT, shm->log, ngx_errno,
"mmap(/dev/zero, MAP_SHARED, %uz) failed", shm->size);
}
@@ -65,7 +65,7 @@ ngx_shm_alloc(ngx_shm_t *shm)
"close(\"/dev/zero\") failed");
}
-return (shm->addr == MAP_FAILED) ? NGX_ERROR : NGX_OK;
+return (shm->addr == (u_char *) MAP_FAILED) ? NGX_ERROR : NGX_OK;
}
[PATCH 0 of 6] various compatibility fixes
Hello! The following patch series fixes some compilation issues as observed on various exotic platforms, notably seen on MINIX, Cygwin, and FreeBSD 2.2.9. -- Maxim Dounin
[PATCH 2 of 6] Configure: added uint8_t and uint16_t detection
# HG changeset patch # User Maxim Dounin # Date 1748209464 -10800 # Mon May 26 00:44:24 2025 +0300 # Node ID 38f1f374523ff2630e59f6a722fd6f9d7f70ac01 # Parent 03f5c84e7e8c05ce16f6f36452e4cb818c85dde6 Configure: added uint8_t and uint16_t detection. This fixes compilation on some old systems (though mostly for consistency with uint32_t and uint64_t). diff --git a/auto/unix b/auto/unix --- a/auto/unix +++ b/auto/unix @@ -645,6 +645,8 @@ ngx_param=NGX_PTR_SIZE; ngx_value=$ngx_s NGX_INCLUDE_AUTO_CONFIG_H="#include \"ngx_auto_config.h\"" +ngx_type="uint8_t"; ngx_types="u_int8_t"; . auto/types/typedef +ngx_type="uint16_t"; ngx_types="u_int16_t"; . auto/types/typedef ngx_type="uint32_t"; ngx_types="u_int32_t"; . auto/types/typedef ngx_type="uint64_t"; ngx_types="u_int64_t"; . auto/types/typedef
