shenjc2013 opened a new issue, #9950:
URL: https://github.com/apache/apisix/issues/9950
### Current Behavior
同一张https通配符证书
使用阿里云CLB 配置七层监听:
所有用户访问全部正常
日志记录： ssl_cipher:ECDHE-RSA-AES128-GCM-SHA256
日志记录 ssl_protocol:TLSv1.2
使用apisix
架构: 阿里云 四层负载均衡 + apisix
apisix跑在k8s集群内部，通过k8s服务发现转发流量，没有使用crd
nginx.conf：
master_process on;
worker_processes auto;
worker_cpu_affinity auto;
error_log /data/logs/apisix-error.log warn;
pid logs/nginx.pid;
worker_rlimit_nofile 600000;
events {
accept_mutex off;
worker_connections 60960;
}
worker_rlimit_core 16G;
worker_shutdown_timeout 240s;
env APISIX_PROFILE;
env KUBERNETES_SERVICE_PORT;
env KUBERNETES_SERVICE_HOST;
lua {
}
stream {
lua_package_path
"$prefix/deps/share/lua/5.1/?.lua;$prefix/deps/share/lua/5.1/?/init.lua;/usr/local/apisix/?.lua;/usr/local/apisix/?/init.lua;;./?.lua;/usr/local/openresty/luajit/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/share/lua/5.1/?/init.lua;/usr/local/openresty/luajit/share/lua/5.1/?.lua;/usr/local/openresty/luajit/share/lua/5.1/?/init.lua;;";
lua_package_cpath
"$prefix/deps/lib64/lua/5.1/?.so;$prefix/deps/lib/lua/5.1/?.so;;./?.so;/usr/local/lib/lua/5.1/?.so;/usr/local/openresty/luajit/lib/lua/5.1/?.so;/usr/local/lib/lua/5.1/loadall.so;";
lua_socket_log_errors off;
lua_max_pending_timers 16384;
lua_max_running_timers 4096;
lua_shared_dict lrucache-lock-stream 10m;
lua_shared_dict etcd-cluster-health-check-stream 10m;
lua_shared_dict plugin-limit-conn-stream 10m;
resolver 172.16.24.10 valid=30 ipv6=on;
resolver_timeout 5;
upstream apisix_backend {
server 127.0.0.1:80;
balancer_by_lua_block {
apisix.stream_balancer_phase()
}
}
init_by_lua_block {
require "resty.core"
apisix = require("apisix")
local dns_resolver = { "172.16.24.10", }
local args = {
dns_resolver = dns_resolver,
}
apisix.stream_init(args)
}
init_worker_by_lua_block {
apisix.stream_init_worker()
}
server {
listen 9100 reuseport;
listen 9200 udp reuseport;
preread_by_lua_block {
apisix.stream_preread_phase()
}
proxy_pass apisix_backend;
set $upstream_sni "apisix_backend";
proxy_ssl_server_name on;
proxy_ssl_name $upstream_sni;
log_by_lua_block {
apisix.stream_log_phase()
}
}
}
http {
lua_package_path
"$prefix/deps/share/lua/5.1/?.lua;$prefix/deps/share/lua/5.1/?/init.lua;/usr/local/apisix/?.lua;/usr/local/apisix/?/init.lua;;./?.lua;/usr/local/openresty/luajit/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/share/lua/5.1/?/init.lua;/usr/local/openresty/luajit/share/lua/5.1/?.lua;/usr/local/openresty/luajit/share/lua/5.1/?/init.lua;;";
lua_package_cpath
"$prefix/deps/lib64/lua/5.1/?.so;$prefix/deps/lib/lua/5.1/?.so;;./?.so;/usr/local/lib/lua/5.1/?.so;/usr/local/openresty/luajit/lib/lua/5.1/?.so;/usr/local/lib/lua/5.1/loadall.so;";
lua_max_pending_timers 16384;
lua_max_running_timers 4096;
lua_shared_dict internal-status 10m;
lua_shared_dict upstream-healthcheck 10m;
lua_shared_dict worker-events 10m;
lua_shared_dict lrucache-lock 10m;
lua_shared_dict balancer-ewma 10m;
lua_shared_dict balancer-ewma-locks 10m;
lua_shared_dict balancer-ewma-last-touched-at 10m;
lua_shared_dict kubernetes 1m;
lua_shared_dict plugin-limit-conn 10m;
lua_shared_dict plugin-limit-req 10m;
lua_shared_dict plugin-limit-count 10m;
lua_shared_dict plugin-limit-count-redis-cluster-slot-lock 1m;
lua_shared_dict prometheus-metrics 10m;
lua_shared_dict plugin-api-breaker 10m;
proxy_cache_path /tmp/disk_cache_one levels=1:2
keys_zone=disk_cache_one:2048m inactive=1d max_size=1G use_temp_path=off;
map $upstream_cache_zone $upstream_cache_zone_info {
disk_cache_one /tmp/disk_cache_one,1:2;
}
lua_ssl_verify_depth 5;
ssl_session_timeout 86400;
underscores_in_headers on;
lua_socket_log_errors off;
proxy_buffer_size 128k;
proxy_buffers 4 256k;
proxy_busy_buffers_size 256k;
resolver 172.16.24.10 valid=30 ipv6=on;
resolver_timeout 5;
lua_http10_buffering off;
lua_regex_match_limit 100000;
lua_regex_cache_max_entries 8192;
log_format main escape=json '{"time_local": "$time_local", "env":
"prod", "proxy_protocol_addr":"$proxy_protocol_addr", "remote_addr":
"$remote_addr", "server_addr": "$server_addr", "host": "$host",
"http_x_forwarded_for": "$http_x_forwarded_for", "request_method":
"$request_method", "uri": "$uri", "args": "$args", "status": "$status",
"body_bytes_sent": "$body_bytes_sent", "request_time": "$request_time",
"http_referer": "$http_referer", "http_user_agent": "$http_user_agent",
"upstream_addr": "$upstream_addr", "upstream_status": "$upstream_status",
"upstream_connect_time": "$upstream_connect_time", "upstream_response_time":
"$upstream_response_time" }';
uninitialized_variable_warn off;
access_log /data/logs/apisix-access.log main buffer=16384 flush=3;
open_file_cache max=1000 inactive=60;
client_max_body_size 2048m;
keepalive_timeout 60s;
client_header_timeout 60s;
client_body_timeout 60s;
send_timeout 10s;
variables_hash_max_size 2048;
server_tokens off;
include mime.types;
charset utf-8;
error_page 500 @50x.html;
real_ip_header X-Real-IP;
real_ip_recursive off;
set_real_ip_from 127.0.0.1;
set_real_ip_from unix:;
upstream apisix_backend {
server 0.0.0.1;
keepalive 320;
keepalive_requests 1000;
keepalive_timeout 60s;
balancer_by_lua_block {
apisix.http_balancer_phase()
}
}
apisix_delay_client_max_body_check on;
apisix_mirror_on_demand on;
init_by_lua_block {
require "resty.core"
apisix = require("apisix")
local dns_resolver = { "172.16.24.10", }
local args = {
dns_resolver = dns_resolver,
}
apisix.http_init(args)
}
init_worker_by_lua_block {
apisix.http_init_worker()
}
exit_worker_by_lua_block {
apisix.http_exit_worker()
}
server {
listen 127.0.0.1:9090;
access_log off;
location / {
content_by_lua_block {
apisix.http_control()
}
}
location @50x.html {
set $from_error_page 'true';
content_by_lua_block {
require("apisix.error_handling").handle_500()
}
}
}
server {
listen 127.0.0.1:9091;
access_log off;
location / {
content_by_lua_block {
local prometheus =
require("apisix.plugins.prometheus.exporter")
prometheus.export_metrics()
}
}
location = /apisix/nginx_status {
allow 127.0.0.0/24;
deny all;
stub_status;
}
}
server {
listen 0.0.0.0:9180;
log_not_found off;
set $upstream_scheme 'http';
set $upstream_host $http_host;
set $upstream_uri '';
location /apisix/admin {
allow 127.0.0.1/24;
allow 0.0.0.0/0;
deny all;
content_by_lua_block {
apisix.http_admin()
}
}
location @50x.html {
set $from_error_page 'true';
content_by_lua_block {
require("apisix.error_handling").handle_500()
}
}
}
server {
listen 0.0.0.0:9080 default_server reuseport;
listen [::]:9080 default_server reuseport;
listen 0.0.0.0:80 default_server reuseport;
listen [::]:80 default_server reuseport;
listen 0.0.0.0:9443 ssl default_server http2 reuseport;
listen [::]:9443 ssl default_server http2 reuseport;
listen 0.0.0.0:443 ssl default_server http2 reuseport;
listen [::]:443 ssl default_server http2 reuseport;
listen 10080 default_server proxy_protocol;
listen 10443 ssl default_server http2 proxy_protocol;
server_name _;
ssl_certificate cert/ssl_PLACE_HOLDER.crt;
ssl_certificate_key cert/ssl_PLACE_HOLDER.key;
ssl_session_cache shared:SSL:20m;
ssl_session_timeout 10m;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
ssl_ciphers
ECDHE-RSA-AES128-GCM-SHA256:ECDHE:ECDH:AES:HIGH:!NULL:!aNULL:!MD5:!ADH:!RC4:!3DES;
ssl_prefer_server_ciphers on;
ssl_session_tickets off;
location = /apisix/nginx_status {
allow 127.0.0.0/24;
deny all;
access_log off;
stub_status;
}
ssl_certificate_by_lua_block {
apisix.http_ssl_phase()
}
proxy_ssl_name $upstream_host;
proxy_ssl_server_name on;
location / {
set $upstream_mirror_uri '';
set $upstream_upgrade '';
set $upstream_connection '';
set $upstream_scheme 'http';
set $upstream_host $http_host;
set $upstream_uri '';
set $ctx_ref '';
set $from_error_page '';
access_by_lua_block {
apisix.http_access_phase()
}
proxy_http_version 1.1;
proxy_set_header Host $upstream_host;
proxy_set_header Upgrade $upstream_upgrade;
proxy_set_header Connection $upstream_connection;
proxy_set_header X-Real-IP $remote_addr;
proxy_pass_header Date;
set $var_x_forwarded_for $remote_addr;
set $var_x_forwarded_proto $scheme;
set $var_x_forwarded_host $host;
set $var_x_forwarded_port $server_port;
if ($http_x_forwarded_for != "") {
set $var_x_forwarded_for "${realip_remote_addr}";
}
if ($http_x_forwarded_host != "") {
set $var_x_forwarded_host $http_x_forwarded_host;
}
if ($http_x_forwarded_port != "") {
set $var_x_forwarded_port $http_x_forwarded_port;
}
proxy_set_header X-Forwarded-For $var_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $var_x_forwarded_proto;
proxy_set_header X-Forwarded-Host $var_x_forwarded_host;
proxy_set_header X-Forwarded-Port $var_x_forwarded_port;
proxy_set_header wanxin-ip $proxy_protocol_addr;
set $upstream_cache_zone off;
set $upstream_cache_key '';
set $upstream_cache_bypass '';
set $upstream_no_cache '';
proxy_cache $upstream_cache_zone;
proxy_cache_valid any 10s;
proxy_cache_min_uses 1;
proxy_cache_methods GET HEAD POST;
proxy_cache_lock_timeout 5s;
proxy_cache_use_stale off;
proxy_cache_key $upstream_cache_key;
proxy_no_cache $upstream_no_cache;
proxy_cache_bypass $upstream_cache_bypass;
proxy_pass $upstream_scheme://apisix_backend$upstream_uri;
mirror /proxy_mirror;
header_filter_by_lua_block {
apisix.http_header_filter_phase()
}
body_filter_by_lua_block {
apisix.http_body_filter_phase()
}
log_by_lua_block {
apisix.http_log_phase()
}
}
location @grpc_pass {
access_by_lua_block {
apisix.grpc_access_phase()
}
grpc_set_header Content-Type application/grpc;
grpc_socket_keepalive on;
grpc_pass $upstream_scheme://apisix_backend;
header_filter_by_lua_block {
apisix.http_header_filter_phase()
}
body_filter_by_lua_block {
apisix.http_body_filter_phase()
}
log_by_lua_block {
apisix.http_log_phase()
}
}
location = /proxy_mirror {
internal;
proxy_connect_timeout 60s;
proxy_read_timeout 60s;
proxy_send_timeout 60s;
proxy_http_version 1.1;
proxy_set_header Host $upstream_host;
proxy_pass $upstream_mirror_uri;
}
location @50x.html {
set $from_error_page 'true';
content_by_lua_block {
require("apisix.error_handling").handle_500()
}
header_filter_by_lua_block {
apisix.http_header_filter_phase()
}
log_by_lua_block {
apisix.http_log_phase()
}
}
}
配置dns解析流量1%-30%到 apisix ，大约4w个IP，其中有2个用户报错
:46#46: *507998 SSL_read() failed (SSL: error:14191044:SSL
routines:tls1_enc:internal error) while waiting for request, client:
42.224.21.155, server: 0.0.0.0:443
非常诡异，用户每分钟会访问1次心跳接口，该问题非常容易复现，可以确定的是证书加密算法、TLS版本、都是一样的，证书文件也是一样的digicert证书厂商下的子公司RapidSSL签发的付费证书，直接购买并配置digicert、亚信的证书也不行，同样的情况
### Expected Behavior
_No response_
### Error Logs
:46#46: *507998 SSL_read() failed (SSL: error:14191044:SSL
routines:tls1_enc:internal error) while waiting for request, client:
42.224.21.155, server: 0.0.0.0:443
### Steps to Reproduce
1. 证书添加完整的证书链，包括中间厂商证书链、rootCA证书链
2.nginx.conf 添加配置 ssl_ecdh_curve prime256v1:X25519:secp521r1:secp384r1;
修改上述配置，还是会报错
3.证书文件也是一样的digicert证书厂商下的子公司RapidSSL签发的付费证书，直接购买并配置digicert、亚信的证书也不行
### Environment
- APISIX version (run `apisix version`):
- Operating system (run `uname -a`):
- OpenResty / Nginx version (run `openresty -V` or `nginx -V`):
- etcd version, if relevant (run `curl
http://127.0.0.1:9090/v1/server_info`):
- APISIX Dashboard version, if relevant:
- Plugin runner version, for issues related to plugin runners:
- LuaRocks version, for installation issues (run `luarocks --version`):
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscr...@apisix.apache.org.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
