zlhgo commented on issue #6184:
URL: https://github.com/apache/apisix/issues/6184#issuecomment-1032135430
@tzssangglass 你好，因为这个问题不是必现的，只有当量大的时候偶尔会出现这个问题。所以我也没有办法让它复现。下面是我的配置文件，再加上前面
ky-ngx-vars.py 插件就是我的全部改动了。
config.yaml
```yaml
apisix:
node_listen: # This style support multiple ports
- 80
ssl:
enable: true
listen: # APISIX listening port in https.
- 443
ssl_protocols: "TLSv1 TLSv1.1 TLSv1.2 TLSv1.3"
admin_key:
- name: admin
key: ****** # using fixed API token has security risk, please update
it when you deploy to production environment
role: admin
admin_listen:
ip: "0.0.0.0"
port: 9180
allow_admin: #
http://nginx.org/en/docs/http/ngx_http_access_module.html#allow
- 127.0.0.0/24 # If we don't set any IP list, then any IP
access is allowed by default.
- 172.16.0.0/16
extra_lua_path: "/usr/local/apisix/extra/?.lua"
nginx_config: # config for render the template to
genarate nginx.conf
error_log: "/home/logs/applog/error.log"
error_log_level: "info" # warn,error
worker_rlimit_nofile: 20480 # the number of files a worker process can
open, should be larger than worker_connections
event:
worker_connections: 10620
http:
access_log: "/dev/stdout"
access_log_format: '$remote_addr - [$remote_addr] - $remote_user
[$time_local] "$request" $status $body_bytes_sent "$http_referer"
"$http_user_agent" $request_length $request_time [$upstream_name]
$upstream_addr $upstream_response_length $upstream_response_time
$upstream_status $req_id $host [$http_client_v] [$http_device_id]
[$http_utm_source] [$http_platform_brand] [$scheme] [$route_name]'
keepalive_timeout: 75s # timeout during which a keep-alive
client connection will stay open on the server side.
client_header_timeout: 120s # timeout for reading client request
header, then 408 (Request Time-out) error is returned to the client
client_body_timeout: 120s # timeout for reading client request
body, then 408 (Request Time-out) error is returned to the client
send_timeout: 10s # timeout for transmitting a response to
the client.then the connection is closed
underscores_in_headers: "on" # default enables the use of underscores
in client request header fields
real_ip_header: "X-Real-IP" #
http://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_header
real_ip_from: #
http://nginx.org/en/docs/http/ngx_http_realip_module.html#set_real_ip_from
- 127.0.0.1
- 'unix:'
#lua_shared_dicts: # add custom shared cache to nginx.conf
# ipc_shared_dict: 100m # custom shared cache, format:
`cache-key: cache-size`
envs: # allow to get a list of environment
variables
- CLUSTER
main_configuration_snippet: |-
# load_module
/usr/local/openresty/nginx/modules/ngx_http_opentracing_module.so;
http_configuration_snippet: |-
tcp_nopush on;
tcp_nodelay on;
reset_timedout_connection on;
keepalive_requests 100;
client_body_temp_path /tmp/client-body;
fastcgi_temp_path /tmp/fastcgi-temp;
proxy_temp_path /tmp/proxy-temp;
proxy_buffering off;
proxy_buffer_size 32k;
proxy_buffers 8 32k;
client_header_buffer_size 1k;
large_client_header_buffers 4 8k;
client_body_buffer_size 10m;
types_hash_max_size 2048;
server_names_hash_max_size 4096;
server_names_hash_bucket_size 64;
proxy_headers_hash_max_size 512;
proxy_headers_hash_bucket_size 64;
variables_hash_bucket_size 256;
ignore_invalid_headers on;
gzip on;
gzip_comp_level 1;
gzip_http_version 1.1;
gzip_min_length 256;
gzip_types application/atom+xml application/javascript
application/x-javascript application/json application/rss+xml
application/vnd.ms-fontobject application/x-font-ttf
application/x-web-app-manifest+json application/xhtml+xml application/xml
font/opentype image/svg+xml image/x-icon text/css text/javascript text/plain
text/x-component;
gzip_proxied any;
gzip_vary on;
# opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so
/usr/local/apisix/conf/opentracing.json;
# opentracing on;
# Reverse proxies can detect if a client provides a X-Request-ID header,
and pass it on to the
# If no such header is provided, it can provide a random value.
map $http_x_request_id $req_id {
default $http_x_request_id;
"" $request_id;
}
log_format log_json escape=json '{"timestamp": "$time_local",'
'"rmt_addr": "$remote_addr",'
'"rmt_user": "$remote_user",'
'"x_forwarded": "$http_x_forwarded_for",'
'"ret_code": "$status",'
'"req_meth": "$request_method",'
'"req_host": "$host",'
'"req_uri": "$request_uri",'
'"req_body": "$request_body",'
'"req_time": "$request_time",'
'"res_time": "$upstream_response_time",'
'"upstream_status": "$upstream_status",'
'"upstream_addr": "$upstream_addr"'
'}';
http_server_configuration_snippet: |
http2_max_field_size 4k;
http2_max_header_size 16k;
http2_max_requests 1000;
http2_max_concurrent_streams 128;
http_server_location_configuration_snippet: |-
if ($host ~* (\.int\.|\.pub\.)) {
# 生产环境不允许访问 int pub 实例
return 403 'deny';
}
set $route_name -;
set $upstream_name -;
proxy_set_header X-Request-ID $req_id;
add_header X-Request-ID $req_id;
client_max_body_size 200m;
# opentracing_propagate_context;
http_admin_configuration_snippet: |-
access_log /home/logs/projectLog/admin.log log_json;
lua_shared_dict:
prometheus-metrics: 100m
etcd:
host: # it's possible to define multiple etcd
hosts addresses of the same etcd cluster.
- "http://172.16.78.1:2379"; # multiple etcd address, if your etcd
cluster enables TLS, please use https scheme,
- "http://172.16.78.2:2379";
- "http://172.16.78.3:2379";
prefix: /apisix/${{CLUSTER}} # apisix configurations
prefix ${{CLUSTER}} = prod-1
timeout: 30 # 30 seconds
plugins: # plugin list (sorted by priority)
- real-ip # priority: 23000
- client-control # priority: 22000
# - ext-plugin-pre-req # priority: 12000
- request-id # priority: 11010
- fault-injection # priority: 11000
- serverless-pre-function # priority: 10000
- batch-requests # priority: 4010
- cors # priority: 4000
- ip-restriction # priority: 3000
- ua-restriction # priority: 2999
- referer-restriction # priority: 2990
- uri-blocker # priority: 2900
# - request-validation # priority: 2800
# - openid-connect # priority: 2599
# - authz-casbin # priority: 2560
# - wolf-rbac # priority: 2555
# - hmac-auth # priority: 2530
- basic-auth # priority: 2520
# - jwt-auth # priority: 2510
# - key-auth # priority: 2500
- consumer-restriction # priority: 2400
# - authz-keycloak # priority: 2000
#- error-log-logger # priority: 1091
- proxy-mirror # priority: 1010
- proxy-cache # priority: 1009
- proxy-rewrite # priority: 1008
- api-breaker # priority: 1005
- limit-conn # priority: 1003
- limit-count # priority: 1002
- limit-req # priority: 1001
#- node-status # priority: 1000
- gzip # priority: 995
- server-info # priority: 990
- traffic-split # priority: 966
- redirect # priority: 900
# - response-rewrite # priority: 899
#- dubbo-proxy # priority: 507
# - grpc-transcode # priority: 506
- prometheus # priority: 500
- echo # priority: 412
# - http-logger # priority: 410
# - sls-logger # priority: 406
# - tcp-logger # priority: 405
# - kafka-logger # priority: 403
# - syslog # priority: 401
# - udp-logger # priority: 400
#- log-rotate # priority: 100
# <- recommend to use priority (0, 100) for your custom plugins
#- skywalking # priority: -1100
- serverless-post-function # priority: -2000
# - ext-plugin-post-req # priority: -3000
# - ky-traffic-control
- ky-ngx-vars
stream_plugins: # sorted by priority
- ip-restriction # priority: 3000
- limit-conn # priority: 1003
- mqtt-proxy # priority: 1000
# <- recommend to use priority (0, 100) for your custom plugins
plugin_attr:
log-rotate:
interval: 3600 # rotate interval (unit: second)
max_kept: 168 # max number of log files will be kept
enable_compression: false # enable log file compression(gzip) or not,
default false
prometheus:
export_uri: /apisix/prometheus/metrics
enable_export_server: true
export_addr:
ip: 0.0.0.0
port: 9091
server-info:
report_interval: 60 # server info report interval (unit: second)
report_ttl: 3600 # live time for server info in etcd (unit: second)
dubbo-proxy:
upstream_multiplex_count: 32
request-id:
snowflake:
enable: false
snowflake_epoc: 1609459200000 # the starting timestamp is expressed
in milliseconds
data_machine_bits: 12 # data machine bit, maximum 31,
because Lua cannot do bit operations greater than 31
sequence_bits: 10 # each machine generates a maximum of
(1 << sequence_bits) serial numbers per millisecond
data_machine_ttl: 30 # live time for data_machine in etcd
(unit: second)
data_machine_interval: 10 # lease renewal interval in etcd
(unit: second)
ky-traffic-control:
ab:
server_url: http://ab-service-online.ky-ab.svc.cluster.local:8080
timeout: 3
```
```
master_process on;
worker_processes auto;
worker_cpu_affinity auto;
# main configuration snippet starts
# load_module
/usr/local/openresty/nginx/modules/ngx_http_opentracing_module.so;
# main configuration snippet ends
error_log /home/logs/applog/error.log info;
pid logs/nginx.pid;
worker_rlimit_nofile 20480;
events {
accept_mutex off;
worker_connections 10620;
}
worker_rlimit_core 16G;
worker_shutdown_timeout 240s;
env APISIX_PROFILE;
env PATH; # for searching external plugin runner's binary
env CLUSTER;
http {
# put extra_lua_path in front of the builtin path
# so user can override the source code
lua_package_path
"/usr/local/apisix/extra/?.lua;$prefix/deps/share/lua/5.1/?.lua;$prefix/deps/share/lua/5.1/?/init.lua;/usr/local/apisix/?.lua;/usr/local/apisix/?/init.lua;;/usr/local/openresty/site/lualib/?.ljbc;/usr/local/openresty/site/lualib/?/init.ljbc;/usr/local/openresty/lualib/?.ljbc;/usr/local/openresty/lualib/?/init.ljbc;/usr/local/openresty/site/lualib/?.lua;/usr/local/openresty/site/lualib/?/init.lua;/usr/local/openresty/lualib/?.lua;/usr/local/openresty/lualib/?/init.lua;./?.lua;/usr/local/openresty/luajit/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/share/lua/5.1/?/init.lua;/usr/local/openresty/luajit/share/lua/5.1/?.lua;/usr/local/openresty/luajit/share/lua/5.1/?/init.lua;";
lua_package_cpath
"$prefix/deps/lib64/lua/5.1/?.so;$prefix/deps/lib/lua/5.1/?.so;;/usr/local/openresty/site/lualib/?.so;/usr/local/openresty/lualib/?.so;./?.so;/usr/local/lib/lua/5.1/?.so;/usr/local/openresty/luajit/lib/lua/5.1/?.so;/usr/local/lib/lua/5.1/loadall.so;/usr/local/openresty/luajit/lib/lua/5.1/?.so;";
lua_max_pending_timers 16384;
lua_max_running_timers 4096;
lua_shared_dict internal-status 10m;
lua_shared_dict plugin-limit-req 10m;
lua_shared_dict plugin-limit-count 10m;
lua_shared_dict prometheus-metrics 100m;
lua_shared_dict plugin-limit-conn 10m;
lua_shared_dict upstream-healthcheck 10m;
lua_shared_dict worker-events 10m;
lua_shared_dict lrucache-lock 10m;
lua_shared_dict balancer-ewma 10m;
lua_shared_dict balancer-ewma-locks 10m;
lua_shared_dict balancer-ewma-last-touched-at 10m;
lua_shared_dict plugin-limit-count-redis-cluster-slot-lock 1m;
lua_shared_dict tracing_buffer 10m; # plugin: skywalking
lua_shared_dict plugin-api-breaker 10m;
lua_shared_dict etcd-cluster-health-check 10m; # etcd health check
# for openid-connect and authz-keycloak plugin
lua_shared_dict discovery 1m; # cache for discovery metadata documents
# for openid-connect plugin
lua_shared_dict jwks 1m; # cache for JWKs
lua_shared_dict introspection 10m; # cache for JWT verification results
# for authz-keycloak
lua_shared_dict access-tokens 1m; # cache for service account access
tokens
# for custom shared dict
# for proxy cache
proxy_cache_path /tmp/disk_cache_one levels=1:2
keys_zone=disk_cache_one:50m inactive=1d max_size=1G use_temp_path=off;
# for proxy cache
map $upstream_cache_zone $upstream_cache_zone_info {
disk_cache_one /tmp/disk_cache_one,1:2;
}
lua_ssl_verify_depth 5;
ssl_session_timeout 86400;
underscores_in_headers on;
lua_socket_log_errors off;
resolver 169.254.20.10;
resolver_timeout 5;
lua_http10_buffering off;
lua_regex_match_limit 100000;
lua_regex_cache_max_entries 8192;
log_format main escape=default '$remote_addr - [$remote_addr] -
$remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer"
"$http_user_agent" $request_length $request_time [$upstream_name]
$upstream_addr $upstream_response_length $upstream_response_time
$upstream_status $req_id $host [$http_client_v] [$http_device_id]
[$http_utm_source] [$http_platform_brand] [$scheme] [$route_name]';
uninitialized_variable_warn off;
access_log /dev/stdout main buffer=16384 flush=3;
open_file_cache max=1000 inactive=60;
client_max_body_size 0;
keepalive_timeout 75s;
client_header_timeout 120s;
client_body_timeout 120s;
send_timeout 10s;
variables_hash_max_size 2048;
server_tokens off;
include mime.types;
charset utf-8;
# error_page
error_page 500 @50x.html;
real_ip_header X-Real-IP;
real_ip_recursive off;
set_real_ip_from 127.0.0.1;
set_real_ip_from unix:;
# http configuration snippet starts
tcp_nopush on;
tcp_nodelay on;
reset_timedout_connection on;
keepalive_requests 100;
client_body_temp_path /tmp/client-body;
fastcgi_temp_path /tmp/fastcgi-temp;
proxy_temp_path /tmp/proxy-temp;
proxy_buffering off;
proxy_buffer_size 32k;
proxy_buffers 8 32k;
client_header_buffer_size 1k;
large_client_header_buffers 4 8k;
client_body_buffer_size 10m;
types_hash_max_size 2048;
server_names_hash_max_size 4096;
server_names_hash_bucket_size 64;
proxy_headers_hash_max_size 512;
proxy_headers_hash_bucket_size 64;
variables_hash_bucket_size 256;
ignore_invalid_headers on;
gzip on;
gzip_comp_level 1;
gzip_http_version 1.1;
gzip_min_length 256;
gzip_types application/atom+xml application/javascript
application/x-javascript application/json application/rss+xml
application/vnd.ms-fontobject application/x-font-ttf
application/x-web-app-manifest+json application/xhtml+xml application/xml
font/opentype image/svg+xml image/x-icon text/css text/javascript text/plain
text/x-component;
gzip_proxied any;
gzip_vary on;
# opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so
/usr/local/apisix/conf/opentracing.json;
# opentracing on;
map $http_x_request_id $req_id {
default $http_x_request_id;
"" $request_id;
}
log_format log_json escape=json '{"timestamp": "$time_local",'
'"rmt_addr": "$remote_addr",'
'"rmt_user": "$remote_user",'
'"x_forwarded": "$http_x_forwarded_for",'
'"ret_code": "$status",'
'"req_meth": "$request_method",'
'"req_host": "$host",'
'"req_uri": "$request_uri",'
'"req_body": "$request_body",'
'"req_time": "$request_time",'
'"res_time": "$upstream_response_time",'
'"upstream_status": "$upstream_status",'
'"upstream_addr": "$upstream_addr"'
'}';
# http configuration snippet ends
upstream apisix_backend {
server 0.0.0.1;
balancer_by_lua_block {
apisix.http_balancer_phase()
}
keepalive 320;
keepalive_requests 1000;
keepalive_timeout 60s;
}
init_by_lua_block {
require "resty.core"
apisix = require("apisix")
local dns_resolver = { "169.254.20.10", }
local args = {
dns_resolver = dns_resolver,
}
apisix.http_init(args)
}
init_worker_by_lua_block {
apisix.http_init_worker()
}
exit_worker_by_lua_block {
apisix.http_exit_worker()
}
server {
listen 127.0.0.1:9090;
access_log off;
location / {
content_by_lua_block {
apisix.http_control()
}
}
location @50x.html {
set $from_error_page 'true';
content_by_lua_block {
require("apisix.error_handling").handle_500()
}
}
}
server {
listen 0.0.0.0:9091;
access_log off;
location / {
content_by_lua_block {
local prometheus = require("apisix.plugins.prometheus")
prometheus.export_metrics()
}
}
location = /apisix/nginx_status {
allow 127.0.0.0/24;
deny all;
stub_status;
}
}
server {
listen 0.0.0.0:9180;
log_not_found off;
# admin configuration snippet starts
access_log /home/logs/projectLog/admin.log log_json;
# admin configuration snippet ends
set $upstream_scheme 'http';
set $upstream_host $http_host;
set $upstream_uri '';
location /apisix/admin {
allow 127.0.0.0/24;
allow 172.16.0.0/16;
deny all;
content_by_lua_block {
apisix.http_admin()
}
}
location @50x.html {
set $from_error_page 'true';
content_by_lua_block {
require("apisix.error_handling").handle_500()
}
}
}
server {
listen 0.0.0.0:80 default_server reuseport;
listen [::]:80 default_server reuseport;
listen 0.0.0.0:443 ssl default_server http2 reuseport;
listen [::]:443 ssl default_server http2 reuseport;
server_name _;
ssl_certificate cert/ssl_PLACE_HOLDER.crt;
ssl_certificate_key cert/ssl_PLACE_HOLDER.key;
ssl_session_cache shared:SSL:20m;
ssl_session_timeout 10m;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
ssl_ciphers
ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
ssl_prefer_server_ciphers on;
ssl_session_tickets off;
# http server configuration snippet starts
http2_max_field_size 4k;
http2_max_header_size 16k;
http2_max_requests 1000;
http2_max_concurrent_streams 128;
# http server configuration snippet ends
location = /apisix/nginx_status {
allow 127.0.0.0/24;
deny all;
access_log off;
stub_status;
}
ssl_certificate_by_lua_block {
apisix.http_ssl_phase()
}
proxy_ssl_name $upstream_host;
proxy_ssl_server_name on;
location / {
set $upstream_mirror_host '';
set $upstream_upgrade '';
set $upstream_connection '';
set $upstream_scheme 'http';
set $upstream_host $http_host;
set $upstream_uri '';
set $ctx_ref '';
set $from_error_page '';
# http server location configuration snippet starts
if ($host ~* (\.int\.|\.pub\.)) {
# 生产环境不允许访问 int pub 实例
return 403 'deny';
}
set $route_name -;
set $upstream_name -;
proxy_set_header X-Request-ID $req_id;
add_header X-Request-ID $req_id;
client_max_body_size 200m;
# opentracing_propagate_context;
# http server location configuration snippet ends
access_by_lua_block {
apisix.http_access_phase()
}
proxy_http_version 1.1;
proxy_set_header Host $upstream_host;
proxy_set_header Upgrade $upstream_upgrade;
proxy_set_header Connection $upstream_connection;
proxy_set_header X-Real-IP $remote_addr;
proxy_pass_header Date;
### the following x-forwarded-* headers is to send to upstream
server
set $var_x_forwarded_for $remote_addr;
set $var_x_forwarded_proto $scheme;
set $var_x_forwarded_host $host;
set $var_x_forwarded_port $server_port;
if ($http_x_forwarded_for != "") {
set $var_x_forwarded_for "${http_x_forwarded_for},
${realip_remote_addr}";
}
if ($http_x_forwarded_host != "") {
set $var_x_forwarded_host $http_x_forwarded_host;
}
if ($http_x_forwarded_port != "") {
set $var_x_forwarded_port $http_x_forwarded_port;
}
proxy_set_header X-Forwarded-For $var_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $var_x_forwarded_proto;
proxy_set_header X-Forwarded-Host $var_x_forwarded_host;
proxy_set_header X-Forwarded-Port $var_x_forwarded_port;
### the following configuration is to cache response content
from upstream server
set $upstream_cache_zone off;
set $upstream_cache_key '';
set $upstream_cache_bypass '';
set $upstream_no_cache '';
proxy_cache $upstream_cache_zone;
proxy_cache_valid any 10s;
proxy_cache_min_uses 1;
proxy_cache_methods GET HEAD POST;
proxy_cache_lock_timeout 5s;
proxy_cache_use_stale off;
proxy_cache_key $upstream_cache_key;
proxy_no_cache $upstream_no_cache;
proxy_cache_bypass $upstream_cache_bypass;
proxy_pass $upstream_scheme://apisix_backend$upstream_uri;
mirror /proxy_mirror;
header_filter_by_lua_block {
apisix.http_header_filter_phase()
}
body_filter_by_lua_block {
apisix.http_body_filter_phase()
}
log_by_lua_block {
apisix.http_log_phase()
}
}
location @grpc_pass {
access_by_lua_block {
apisix.grpc_access_phase()
}
grpc_set_header Content-Type application/grpc;
grpc_socket_keepalive on;
grpc_pass $upstream_scheme://apisix_backend;
header_filter_by_lua_block {
apisix.http_header_filter_phase()
}
body_filter_by_lua_block {
apisix.http_body_filter_phase()
}
log_by_lua_block {
apisix.http_log_phase()
}
}
location = /proxy_mirror {
internal;
if ($upstream_mirror_host = "") {
return 200;
}
proxy_http_version 1.1;
proxy_set_header Host $upstream_host;
proxy_pass $upstream_mirror_host$request_uri;
}
location @50x.html {
set $from_error_page 'true';
content_by_lua_block {
require("apisix.error_handling").handle_500()
}
header_filter_by_lua_block {
apisix.http_header_filter_phase()
}
log_by_lua_block {
apisix.http_log_phase()
}
}
}
# http end configuration snippet starts
# http end configuration snippet ends
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
