Sorry, I accidentally submitted an incomplete version of the patch. Here is the corrected version.
# HG changeset patch # User Johannes Baiter <johannes.bai...@bsb-muenchen.de> # Date 1519312196 -3600 # Thu Feb 22 16:09:56 2018 +0100 # Node ID 6c1e7ddade7dd75677cb2821f346e862866ba4e3 # Parent aa60f5799a4cbfc6887ef099c588e84a75c69785 Access log: support for disabling escaping. diff -r aa60f5799a4c -r 6c1e7ddade7d src/http/modules/ngx_http_log_module.c --- a/src/http/modules/ngx_http_log_module.c Thu Feb 22 12:42:29 2018 +0300 +++ b/src/http/modules/ngx_http_log_module.c Thu Feb 22 16:09:56 2018 +0100 @@ -90,6 +90,13 @@ } ngx_http_log_var_t; +typedef enum { + ngx_http_log_escape_default = 0, + ngx_http_log_escape_json, + ngx_http_log_escape_none +} ngx_http_log_escape_t; + + static void ngx_http_log_write(ngx_http_request_t *r, ngx_http_log_t *log, u_char *buf, size_t len); static ssize_t ngx_http_log_script_write(ngx_http_request_t *r, @@ -126,12 +133,16 @@ ngx_http_log_op_t *op); static ngx_int_t ngx_http_log_variable_compile(ngx_conf_t *cf, - ngx_http_log_op_t *op, ngx_str_t *value, ngx_uint_t json); + ngx_http_log_op_t *op, ngx_str_t *value, ngx_http_log_escape_t escape); static size_t ngx_http_log_variable_getlen(ngx_http_request_t *r, uintptr_t data); static u_char *ngx_http_log_variable(ngx_http_request_t *r, u_char *buf, ngx_http_log_op_t *op); static uintptr_t ngx_http_log_escape(u_char *dst, u_char *src, size_t size); +static size_t ngx_http_log_unescaped_variable_getlen(ngx_http_request_t *r, + uintptr_t data); +static u_char *ngx_http_log_unescaped_variable(ngx_http_request_t *r, u_char *buf, + ngx_http_log_op_t *op); static size_t ngx_http_log_json_variable_getlen(ngx_http_request_t *r, uintptr_t data); static u_char *ngx_http_log_json_variable(ngx_http_request_t *r, u_char *buf, @@ -905,7 +916,7 @@ static ngx_int_t ngx_http_log_variable_compile(ngx_conf_t *cf, ngx_http_log_op_t *op, - ngx_str_t *value, ngx_uint_t json) + ngx_str_t *value, ngx_http_log_escape_t escape) { ngx_int_t index; @@ -916,10 +927,14 @@ op->len = 0; - if (json) { + if (escape == ngx_http_log_escape_json) { op->getlen = ngx_http_log_json_variable_getlen; op->run = ngx_http_log_json_variable; + } else if (escape == ngx_http_log_escape_none) { + op->getlen = ngx_http_log_unescaped_variable_getlen; + op->run = ngx_http_log_unescaped_variable; + } else { op->getlen = ngx_http_log_variable_getlen; op->run = ngx_http_log_variable; @@ -1033,6 +1048,38 @@ static size_t +ngx_http_log_unescaped_variable_getlen(ngx_http_request_t *r, uintptr_t data) +{ + ngx_http_variable_value_t *value; + + value = ngx_http_get_indexed_variable(r, data); + + if (value == NULL || value->not_found) { + return 0; + } + + value->escape = 0; + return value->len; +} + + +static u_char * +ngx_http_log_unescaped_variable(ngx_http_request_t *r, u_char *buf, + ngx_http_log_op_t *op) +{ + ngx_http_variable_value_t *value; + + value = ngx_http_get_indexed_variable(r, op->data); + + if (value == NULL || value->not_found) { + return buf; + } + + return ngx_cpymem(buf, value->data, value->len); +} + + +static size_t ngx_http_log_json_variable_getlen(ngx_http_request_t *r, uintptr_t data) { uintptr_t len; @@ -1536,19 +1583,21 @@ size_t i, len; ngx_str_t *value, var; ngx_int_t *flush; - ngx_uint_t bracket, json; + ngx_uint_t bracket; ngx_http_log_op_t *op; ngx_http_log_var_t *v; + ngx_http_log_escape_t escape; - json = 0; + escape = ngx_http_log_escape_default; value = args->elts; if (s < args->nelts && ngx_strncmp(value[s].data, "escape=", 7) == 0) { data = value[s].data + 7; if (ngx_strcmp(data, "json") == 0) { - json = 1; - + escape = ngx_http_log_escape_json; + } else if (ngx_strcmp(data, "none") == 0) { + escape = ngx_http_log_escape_none; } else if (ngx_strcmp(data, "default") != 0) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "unknown log format escaping \"%s\"", data); @@ -1636,7 +1685,7 @@ } } - if (ngx_http_log_variable_compile(cf, op, &var, json) + if (ngx_http_log_variable_compile(cf, op, &var, escape) != NGX_OK) { return NGX_CONF_ERROR; diff -r aa60f5799a4c -r 6c1e7ddade7d src/stream/ngx_stream_log_module.c --- a/src/stream/ngx_stream_log_module.c Thu Feb 22 12:42:29 2018 +0300 +++ b/src/stream/ngx_stream_log_module.c Thu Feb 22 16:09:56 2018 +0100 @@ -89,6 +89,13 @@ } ngx_stream_log_var_t; +typedef enum { + ngx_stream_log_escape_default = 0, + ngx_stream_log_escape_json, + ngx_stream_log_escape_none +} ngx_stream_log_escape_t; + + static void ngx_stream_log_write(ngx_stream_session_t *s, ngx_stream_log_t *log, u_char *buf, size_t len); static ssize_t ngx_stream_log_script_write(ngx_stream_session_t *s, @@ -106,12 +113,16 @@ static void ngx_stream_log_flush_handler(ngx_event_t *ev); static ngx_int_t ngx_stream_log_variable_compile(ngx_conf_t *cf, - ngx_stream_log_op_t *op, ngx_str_t *value, ngx_uint_t json); + ngx_stream_log_op_t *op, ngx_str_t *value, ngx_stream_log_escape_t escape); static size_t ngx_stream_log_variable_getlen(ngx_stream_session_t *s, uintptr_t data); static u_char *ngx_stream_log_variable(ngx_stream_session_t *s, u_char *buf, ngx_stream_log_op_t *op); static uintptr_t ngx_stream_log_escape(u_char *dst, u_char *src, size_t size); +static size_t ngx_stream_log_unescaped_variable_getlen(ngx_stream_session_t *s, + uintptr_t data); +static u_char *ngx_stream_log_unescaped_variable(ngx_stream_session_t *s, + u_char *buf, ngx_stream_log_op_t *op); static size_t ngx_stream_log_json_variable_getlen(ngx_stream_session_t *s, uintptr_t data); static u_char *ngx_stream_log_json_variable(ngx_stream_session_t *s, @@ -682,7 +693,7 @@ static ngx_int_t ngx_stream_log_variable_compile(ngx_conf_t *cf, ngx_stream_log_op_t *op, - ngx_str_t *value, ngx_uint_t json) + ngx_str_t *value, ngx_stream_log_escape_t escape) { ngx_int_t index; @@ -693,10 +704,14 @@ op->len = 0; - if (json) { + if (escape == ngx_stream_log_escape_json) { op->getlen = ngx_stream_log_json_variable_getlen; op->run = ngx_stream_log_json_variable; + } else if (escape == ngx_stream_log_escape_none) { + op->getlen = ngx_stream_log_unescaped_variable_getlen; + op->run = ngx_stream_log_unescaped_variable; + } else { op->getlen = ngx_stream_log_variable_getlen; op->run = ngx_stream_log_variable; @@ -811,6 +826,38 @@ static size_t +ngx_stream_log_unescaped_variable_getlen(ngx_stream_session_t *s, uintptr_t data) +{ + ngx_stream_variable_value_t *value; + + value = ngx_stream_get_indexed_variable(s, data); + + if (value == NULL || value->not_found) { + return 0; + } + + value->escape = 0; + return value->len; +} + + +static u_char * +ngx_stream_log_unescaped_variable(ngx_stream_session_t *s, u_char *buf, + ngx_stream_log_op_t *op) +{ + ngx_stream_variable_value_t *value; + + value = ngx_stream_get_indexed_variable(s, op->data); + + if (value == NULL || value->not_found) { + return buf; + } + + return ngx_cpymem(buf, value->data, value->len); +} + + +static size_t ngx_stream_log_json_variable_getlen(ngx_stream_session_t *s, uintptr_t data) { uintptr_t len; @@ -1265,17 +1312,21 @@ size_t i, len; ngx_str_t *value, var; ngx_int_t *flush; - ngx_uint_t bracket, json; + ngx_uint_t bracket; ngx_stream_log_op_t *op; + ngx_stream_log_escape_t escape; - json = 0; + escape = ngx_stream_log_escape_default; value = args->elts; if (s < args->nelts && ngx_strncmp(value[s].data, "escape=", 7) == 0) { data = value[s].data + 7; if (ngx_strcmp(data, "json") == 0) { - json = 1; + escape = ngx_stream_log_escape_json; + + } else if (ngx_strcmp(data, "none") == 0) { + escape = ngx_stream_log_escape_none; } else if (ngx_strcmp(data, "default") != 0) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, @@ -1350,7 +1401,7 @@ goto invalid; } - if (ngx_stream_log_variable_compile(cf, op, &var, json) + if (ngx_stream_log_variable_compile(cf, op, &var, escape) != NGX_OK) { return NGX_CONF_ERROR; >>> "Johannes Baiter" <johannes.bai...@bsb-muenchen.de> 2/22/2018 5:08 PM >>> This patch adds a "none" option for the "escape" parameter on the "log_format" directive. Selecting this option will perform no escaping of variables in the access log output and write the variable to the file as-is. The use case for this is more freedom for custom logging setups. In our case, we use structured logging in JSON. For this, we build a JSON string in a Lua script that is then stored in a nginx variable. The log_format in this case is simply the variable: # The script sets the $json_log variable log_by_lua_file /etc/nginx/lua/scripts/json_logging.lua; log_format json $json_log; access_log /local/nginx/log/access.log json; error_log /local/nginx/log/error.log; Both "default" and "json" do not work in this case, since either will result in invalid JSON output. Updated test suites for both ngx_http and ngx_stream are provided in the second patch. # HG changeset patch # User Johannes Baiter <johannes.bai...@bsb-muenchen.de> # Date 1519312196 -3600 # Thu Feb 22 16:09:56 2018 +0100 # Node ID 267e08661efcb58a9b4cd5fd941486b88a37f4f4 # Parent aa60f5799a4cbfc6887ef099c588e84a75c69785 Access log: support for disabling escaping. diff -r aa60f5799a4c -r 267e08661efc src/http/modules/ngx_http_log_module.c --- a/src/http/modules/ngx_http_log_module.c Thu Feb 22 12:42:29 2018 +0300 +++ b/src/http/modules/ngx_http_log_module.c Thu Feb 22 16:09:56 2018 +0100 @@ -90,6 +90,13 @@ } ngx_http_log_var_t; +typedef enum { + ngx_http_log_escape_default = 0, + ngx_http_log_escape_json, + ngx_http_log_escape_none +} ngx_http_log_escape_t; + + static void ngx_http_log_write(ngx_http_request_t *r, ngx_http_log_t *log, u_char *buf, size_t len); static ssize_t ngx_http_log_script_write(ngx_http_request_t *r, @@ -126,12 +133,16 @@ ngx_http_log_op_t *op); static ngx_int_t ngx_http_log_variable_compile(ngx_conf_t *cf, - ngx_http_log_op_t *op, ngx_str_t *value, ngx_uint_t json); + ngx_http_log_op_t *op, ngx_str_t *value, ngx_http_log_escape_t escape); static size_t ngx_http_log_variable_getlen(ngx_http_request_t *r, uintptr_t data); static u_char *ngx_http_log_variable(ngx_http_request_t *r, u_char *buf, ngx_http_log_op_t *op); static uintptr_t ngx_http_log_escape(u_char *dst, u_char *src, size_t size); +static size_t ngx_http_log_unescaped_variable_getlen(ngx_http_request_t *r, + uintptr_t data); +static u_char *ngx_http_log_unescaped_variable(ngx_http_request_t *r, u_char *buf, + ngx_http_log_op_t *op); static size_t ngx_http_log_json_variable_getlen(ngx_http_request_t *r, uintptr_t data); static u_char *ngx_http_log_json_variable(ngx_http_request_t *r, u_char *buf, @@ -905,7 +916,7 @@ static ngx_int_t ngx_http_log_variable_compile(ngx_conf_t *cf, ngx_http_log_op_t *op, - ngx_str_t *value, ngx_uint_t json) + ngx_str_t *value, ngx_http_log_escape_t escape) { ngx_int_t index; @@ -916,10 +927,12 @@ op->len = 0; - if (json) { + if (escape == ngx_http_log_escape_json) { op->getlen = ngx_http_log_json_variable_getlen; op->run = ngx_http_log_json_variable; - + } else if (escape == ngx_http_log_escape_none) { + op->getlen = ngx_http_log_unescaped_variable_getlen; + op->run = ngx_http_log_unescaped_variable; } else { op->getlen = ngx_http_log_variable_getlen; op->run = ngx_http_log_variable; @@ -1033,6 +1046,38 @@ static size_t +ngx_http_log_unescaped_variable_getlen(ngx_http_request_t *r, uintptr_t data) +{ + ngx_http_variable_value_t *value; + + value = ngx_http_get_indexed_variable(r, data); + + if (value == NULL || value->not_found) { + return 0; + } + + value->escape = 0; + return value->len; +} + + +static u_char * +ngx_http_log_unescaped_variable(ngx_http_request_t *r, u_char *buf, + ngx_http_log_op_t *op) +{ + ngx_http_variable_value_t *value; + + value = ngx_http_get_indexed_variable(r, op->data); + + if (value == NULL || value->not_found) { + return buf; + } + + return ngx_cpymem(buf, value->data, value->len); +} + + +static size_t ngx_http_log_json_variable_getlen(ngx_http_request_t *r, uintptr_t data) { uintptr_t len; @@ -1536,19 +1581,21 @@ size_t i, len; ngx_str_t *value, var; ngx_int_t *flush; - ngx_uint_t bracket, json; + ngx_uint_t bracket; ngx_http_log_op_t *op; ngx_http_log_var_t *v; + ngx_http_log_escape_t escape; - json = 0; + escape = ngx_http_log_escape_default; value = args->elts; if (s < args->nelts && ngx_strncmp(value[s].data, "escape=", 7) == 0) { data = value[s].data + 7; if (ngx_strcmp(data, "json") == 0) { - json = 1; - + escape = ngx_http_log_escape_json; + } else if (ngx_strcmp(data, "none") == 0) { + escape = ngx_http_log_escape_none; } else if (ngx_strcmp(data, "default") != 0) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "unknown log format escaping \"%s\"", data); @@ -1636,7 +1683,7 @@ } } - if (ngx_http_log_variable_compile(cf, op, &var, json) + if (ngx_http_log_variable_compile(cf, op, &var, escape) != NGX_OK) { return NGX_CONF_ERROR; diff -r aa60f5799a4c -r 267e08661efc src/stream/ngx_stream_log_module.c --- a/src/stream/ngx_stream_log_module.c Thu Feb 22 12:42:29 2018 +0300 +++ b/src/stream/ngx_stream_log_module.c Thu Feb 22 16:09:56 2018 +0100 @@ -89,6 +89,13 @@ } ngx_stream_log_var_t; +typedef enum { + ngx_stream_log_escape_default = 0, + ngx_stream_log_escape_json, + ngx_stream_log_escape_none +} ngx_stream_log_escape_t; + + static void ngx_stream_log_write(ngx_stream_session_t *s, ngx_stream_log_t *log, u_char *buf, size_t len); static ssize_t ngx_stream_log_script_write(ngx_stream_session_t *s, @@ -106,7 +113,7 @@ static void ngx_stream_log_flush_handler(ngx_event_t *ev); static ngx_int_t ngx_stream_log_variable_compile(ngx_conf_t *cf, - ngx_stream_log_op_t *op, ngx_str_t *value, ngx_uint_t json); + ngx_stream_log_op_t *op, ngx_str_t *value, ngx_stream_log_escape_t escape); static size_t ngx_stream_log_variable_getlen(ngx_stream_session_t *s, uintptr_t data); static u_char *ngx_stream_log_variable(ngx_stream_session_t *s, u_char *buf, @@ -682,7 +689,7 @@ static ngx_int_t ngx_stream_log_variable_compile(ngx_conf_t *cf, ngx_stream_log_op_t *op, - ngx_str_t *value, ngx_uint_t json) + ngx_str_t *value, ngx_stream_log_escape_t escape) { ngx_int_t index; @@ -693,10 +700,14 @@ op->len = 0; - if (json) { + if (escape == ngx_stream_log_escape_json) { op->getlen = ngx_stream_log_json_variable_getlen; op->run = ngx_stream_log_json_variable; + } else if (escape == ngx_stream_log_escape_none) { + op->getlen = ngx_stream_log_unescaped_variable_getlen; + op->run = ngx_stream_log_unescaped_variable; + } else { op->getlen = ngx_stream_log_variable_getlen; op->run = ngx_stream_log_variable; @@ -811,6 +822,38 @@ static size_t +ngx_stream_log_unescaped_variable_getlen(ngx_stream_session_t *s, uintptr_t data) +{ + ngx_stream_variable_value_t *value; + + value = ngx_stream_get_indexed_variable(s, data); + + if (value == NULL || value->not_found) { + return 0; + } + + value->escape = 0; + return value->len; +} + + +static u_char * +ngx_http_log_unescaped_variable(ngx_stream_session_t *s, u_char *buf, + ngx_stream_log_op_t *op) +{ + ngx_stream_variable_value_t *value; + + value = ngx_stream_get_indexed_variable(s, op->data); + + if (value == NULL || value->not_found) { + return buf; + } + + return ngx_cpymem(buf, value->data, value->len); +} + + +static size_t ngx_stream_log_json_variable_getlen(ngx_stream_session_t *s, uintptr_t data) { uintptr_t len; @@ -1265,17 +1308,21 @@ size_t i, len; ngx_str_t *value, var; ngx_int_t *flush; - ngx_uint_t bracket, json; + ngx_uint_t bracket; ngx_stream_log_op_t *op; + ngx_stream_log_escape_t escape; - json = 0; + escape = ngx_stream_log_escape_default; value = args->elts; if (s < args->nelts && ngx_strncmp(value[s].data, "escape=", 7) == 0) { data = value[s].data + 7; if (ngx_strcmp(data, "json") == 0) { - json = 1; + escape = ngx_stream_log_escape_json; + + } else if (ngx_strcmp(data, "none") == 0) { + escape = ngx_stream_log_escape_none; } else if (ngx_strcmp(data, "default") != 0) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, @@ -1350,7 +1397,7 @@ goto invalid; } - if (ngx_stream_log_variable_compile(cf, op, &var, json) + if (ngx_stream_log_variable_compile(cf, op, &var, escape) != NGX_OK) { return NGX_CONF_ERROR; _______________________________________________ nginx-devel mailing list nginx-devel@nginx.org http://mailman.nginx.org/mailman/listinfo/nginx-devel _______________________________________________ nginx-devel mailing list nginx-devel@nginx.org http://mailman.nginx.org/mailman/listinfo/nginx-devel