This is an automated email from the ASF dual-hosted git repository.
monkeydluffy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new 02ee09ed9 ci: tcp logger with real logger server (#9392)
02ee09ed9 is described below
commit 02ee09ed99dba9024ecef1714da57b3d7359a2a1
Author: Sn0rt <[email protected]>
AuthorDate: Sat May 6 09:25:20 2023 +0800
ci: tcp logger with real logger server (#9392)
---
ci/pod/docker-compose.plugin.yml | 13 ++++
ci/pod/vector/vector.toml | 53 +++++++++++++
t/certs/vector_logs_ca.crt | 21 +++++
t/certs/vector_logs_ca.key | 27 +++++++
t/certs/vector_logs_server.crt | 19 +++++
t/certs/vector_logs_server.key | 27 +++++++
t/plugin/tcp-logger.t | 164 ++++++++++++++++++++++++---------------
7 files changed, 260 insertions(+), 64 deletions(-)
diff --git a/ci/pod/docker-compose.plugin.yml b/ci/pod/docker-compose.plugin.yml
index 7647a1ee9..f4e391698 100644
--- a/ci/pod/docker-compose.plugin.yml
+++ b/ci/pod/docker-compose.plugin.yml
@@ -312,9 +312,22 @@ services:
ports:
- '8888:8080'
+ vector:
+ image: timberio/vector:0.29.1-debian
+ container_name: vector
+ volumes:
+ - ./ci/pod/vector:/etc/vector/
+ - ./t/certs:/certs
+ ports:
+ - '3000:3000'
+ - '43000:43000'
+ networks:
+ vector_net:
+
networks:
apisix_net:
kafka_net:
skywalk_net:
rocketmq_net:
opa_net:
+ vector_net:
diff --git a/ci/pod/vector/vector.toml b/ci/pod/vector/vector.toml
new file mode 100644
index 000000000..0e02e0fd2
--- /dev/null
+++ b/ci/pod/vector/vector.toml
@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[sources.log-from-tcp]
+type = "socket"
+address = "0.0.0.0:3000"
+host_key = "host"
+mode = "tcp"
+port_key = "port"
+shutdown_timeout_secs = 30
+socket_file_mode = 511
+
+[sources.log-from-tls]
+type = "socket"
+address = "0.0.0.0:43000"
+host_key = "host"
+mode = "tcp"
+port_key = "port"
+tls.enabled = true
+tls.verify = true
+tls.ca_file = "/certs/vector_logs_ca.crt"
+tls.crt_file = "/certs/vector_logs_server.crt"
+tls.key_file = "/certs/vector_logs_server.key"
+
+[sinks.log-2-console]
+inputs = [ "log-from-tcp", "log-from-tls" ]
+type = "console"
+encoding.codec = "json"
+
+[sinks.log-2-tcp-file]
+inputs = [ "log-from-tcp" ]
+type = "file"
+encoding.codec = "text"
+path = "/etc/vector/tcp.log"
+
+[sinks.tls-log-2-file]
+inputs = [ "log-from-tls" ]
+type = "file"
+encoding.codec = "json"
+path = "/etc/vector/tls-datas.log"
diff --git a/t/certs/vector_logs_ca.crt b/t/certs/vector_logs_ca.crt
new file mode 100644
index 000000000..42e3659f6
--- /dev/null
+++ b/t/certs/vector_logs_ca.crt
@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDazCCAlOgAwIBAgIUa34rzhtYT21pC8NwNIYf3phFciQwDQYJKoZIhvcNAQEL
+BQAwRTELMAkGA1UEBhMCVVMxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMzA0MjQxMzE2NDNaFw0yMzA1
+MjQxMzE2NDNaMEUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
+HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB
+AQUAA4IBDwAwggEKAoIBAQC84FXe/8ofZB2rj5TPHasXdiBEbTCv04ti/lV92WOC
+MrHLKibI2+kI3YRQXaR5/1F2bXfROUhdRgkB8NOSM4WbaD1mtr/7mW2Tatxplxsp
+1s0zmHHl5v0VYwBahcUs6nlSe19dgfrj4s0Wn7p4E7iSq/UDAs+We/dQowusQTVs
+Q2ZhjDlFY22CV/oyCYsNq3ORRgwZRm9cmVmUUF7GX70yjT1KvLkFjc7y1vwi8XJY
+ADhw/hjtEzAOkxdUai84+jyhpQYQWMOgrlP1DXnZw1bNKqo6NTkMzfNCS+ul5PMs
+Noyxcw1iyGW6Bm81LANsnMM7BLhPQATShmW7O83WUJ4vAgMBAAGjUzBRMB0GA1Ud
+DgQWBBRdFCb//WETC8mDxg/75e+RoVNoDjAfBgNVHSMEGDAWgBRdFCb//WETC8mD
+xg/75e+RoVNoDjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQC8
+cKOagO+SVJBMzJppm4uGdSM6TJ2wbkn6K5/eOtZmYdKtW6fkAC9tf0DR7dVP1DUk
+24lS+atR1Oe7SukxJyd+NafCZ61uf+zrMC3wgBGnufrbPWaDDVxi6c3I0I+WNaCk
+DHHY+9UtjvSboWKG1yuEExPN6aDeytbpscG1DNi7l96Ac3Yzs007SFljA7NBrf65
+So9SZYSdJVC/JrOnfK2HZPeAqvoyUO5JsCh02q5AskxTqfBGy6VUVQO5mN8bxYHV
+GG5XD46rpwQYNT2bWWRF5d0bRv7ecNkCoupm6hCQROg4FZHGPnqHGqDTcgCLZ59e
+8rHh2gsDMMNYvSMTi+0N
+-----END CERTIFICATE-----
diff --git a/t/certs/vector_logs_ca.key b/t/certs/vector_logs_ca.key
new file mode 100644
index 000000000..e36b69354
--- /dev/null
+++ b/t/certs/vector_logs_ca.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAvOBV3v/KH2Qdq4+Uzx2rF3YgRG0wr9OLYv5VfdljgjKxyyom
+yNvpCN2EUF2kef9Rdm130TlIXUYJAfDTkjOFm2g9Zra/+5ltk2rcaZcbKdbNM5hx
+5eb9FWMAWoXFLOp5UntfXYH64+LNFp+6eBO4kqv1AwLPlnv3UKMLrEE1bENmYYw5
+RWNtglf6MgmLDatzkUYMGUZvXJlZlFBexl+9Mo09Sry5BY3O8tb8IvFyWAA4cP4Y
+7RMwDpMXVGovOPo8oaUGEFjDoK5T9Q152cNWzSqqOjU5DM3zQkvrpeTzLDaMsXMN
+YshlugZvNSwDbJzDOwS4T0AE0oZluzvN1lCeLwIDAQABAoIBADM7ou9fcQM80/OC
+efoIcS1nBG+rMqau+kM6/BOsERrzB1k1sNmRFVArTkXCcOgKwp0eKn8dS6zJX44g
+NjOVOCukhetDrSXhQ2DWfr1BmMOrmXPiaRrUolfXx/PGD2sUmx4tivvBUz3Xeowl
+fZ4us0VN0aMkcwy9yaMc5wCtm4Em+uMrUIvWSAl3ji09oG4NNBQHUsEWJoRMZ/AG
+GQowc7Ga850ybZlza1uWh29a3bbQqEwHExJwiCISv25PJ/xQLqH65biB4MU+ym17
+Ou/MDn9cYndxBal/XI4R7HbeIjMgw2XxwXiiDOuKAn5TlCzHmySRXFj1BoT8xoXa
+vTXVlAkCgYEA+nc2GiainyW0MAASX53Ue5zsFh4T2CaA4TTHeXEK22rL1Sz3LsbX
+ymBqCcNwbcSTYUzBsf6YzSsPLUwIzBGEN0p5Ywts5KtWavAxllBj2MOTP4yQfLvh
+AxOq94hqrDLMs/g7LkFrfspYMCXmegGjjXGuqirKbigXkFVQkvOUcwUCgYEAwQy8
+kl2+deq1OD9rJId596nDx6JVLBt7/VP4dOOaS2/YQeFnUdM1xMM+zovEZb3UZMpp
+8yhRE7hB7Fm0688yd+F7GpFC49LyVTitZcaIV7nMnXjJQQ27WyiAZoRKHt1gP4io
+OCZAaOEJRbGJcWR3sSPHfX93R+xEtFNAexb/eqMCgYEA8NDV7+bdzO7PhKdNAyoZ
+NpD2XX2lztmWaPH6KMWLjtPsD5cgQpVkvWxeB+0lmCS9H3xRb/Y+rGWOPhsxCiR9
+Xzv34kcF+AbVHBS9WK0Kk0vXs+5Ord9mxTKP21gKWG6vawpsvFiiJlIe4IxQQVZ6
+DnETYwGpiKh7n4an5eLVBJECgYEAnviuEJnBzbiJotgWku49MgVKg4raOIgpgmMz
+po4G8TgZDadgPbGABZgCkHPoNyArVxSYSvRYT7TcFJWKtuTY2n+DsE0OmC2OAT+7
+CqSCgjsulD5y/G8iad7gXYtyvhfuumL+o75cLAGkcQ/R7t6c8fJUxLPCtieKLDSi
+VLqLh6ECgYAlk8O5Rz2bSje4F+b0PZLAGjQRytpTjrgVpxwJ4bBXYeqhfd+3Fi8s
+OraFx+kj/YOOFkk5uu75/fbccEY1DG0nmWUR0pjHM+QndB4rpkSxtp+pfVo2nRn0
+pAY8ep+TFRLwmy7ZXpOFPYlGPwx+rjSm9vk9EJYjxZE8YYldiBBKHw==
+-----END RSA PRIVATE KEY-----
diff --git a/t/certs/vector_logs_server.crt b/t/certs/vector_logs_server.crt
new file mode 100644
index 000000000..95bb51bec
--- /dev/null
+++ b/t/certs/vector_logs_server.crt
@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDETCCAfkCFEynFsv9L6bzyJJVmjoaKuCoYZwkMA0GCSqGSIb3DQEBCwUAMEUx
+CzAJBgNVBAYTAlVTMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
+cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjMwNDI0MTMxNzAwWhcNMjQwNDIzMTMx
+NzAwWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE
+CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEAtGvdwIHuk6k3vphBnGIdtlZ/6ImHSVBsNHz5y6E9X31a88EH
+wtnxT5Ang8K6Y4pQt+LsjhI0NdUY2skiKDnGpo2IkaFAn9nERQ1GJstIHr7ltal6
+ureV4n/Na/T6n6GPnwD4+P86XvpIwFtJZujYr2tUl4qm/t1P7zHjB/UsF9G6H/aN
+oCsDkG3a7+b8uWAZLkyHS4RLF3pG6pDWns8/vC/P9nTT7o3Ha2DV7TPaY0hlsXf6
+0/SCSm7EonnVVwhnKyy5Z0FsCXClg7weN4ZKPb+ypF0o0/LLqw481lbSfAu5kpjE
+r/rHpsQonRbQrcrD9xovXmw2vdk/2jJn6wpFQwIDAQABMA0GCSqGSIb3DQEBCwUA
+A4IBAQBv9e8gsD75iySf+pam11JUujjL0gpqdzY72CKo4abYX5NZhMiBs6OCKicz
+EedR/EgRY+26RMThKC0zSy3hOO6SKPw03FLsV2B8ooDzaOa4l3F/E6NQ5yNDoK+K
+lT1G85fW3bQWtNoB8aa/r1/eExZy3kZF8GSl+/BvwLtOwtGXMO0Y1URo81Dl0da+
+F2yv6ZGziEYIWYTUK3kxOpe0Sl4wHz33olWoli2qpYlSndUUIWoVYJr4gtH/xTEV
+GHxdOhxcfyMNi6ceYG4HGWyKRFR9TJAU+PRBxHI8UUpg+BG3/DQmfA5+7xgAws37
+dEVsm725hta8vPUSMSAdRrArBlh+
+-----END CERTIFICATE-----
diff --git a/t/certs/vector_logs_server.key b/t/certs/vector_logs_server.key
new file mode 100644
index 000000000..54035d4fa
--- /dev/null
+++ b/t/certs/vector_logs_server.key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAtGvdwIHuk6k3vphBnGIdtlZ/6ImHSVBsNHz5y6E9X31a88EH
+wtnxT5Ang8K6Y4pQt+LsjhI0NdUY2skiKDnGpo2IkaFAn9nERQ1GJstIHr7ltal6
+ureV4n/Na/T6n6GPnwD4+P86XvpIwFtJZujYr2tUl4qm/t1P7zHjB/UsF9G6H/aN
+oCsDkG3a7+b8uWAZLkyHS4RLF3pG6pDWns8/vC/P9nTT7o3Ha2DV7TPaY0hlsXf6
+0/SCSm7EonnVVwhnKyy5Z0FsCXClg7weN4ZKPb+ypF0o0/LLqw481lbSfAu5kpjE
+r/rHpsQonRbQrcrD9xovXmw2vdk/2jJn6wpFQwIDAQABAoIBAQCB24lV/6759Le8
+pNXEexIrpQKXGjWXXR0kgjdAiyMjUZRfETZG1prKy1TFjyiccHc8g0YD07JkdKZZ
+Ap9lGICUbBY5yzg6VYDguncdgP69smSfZgaB0ZU92wK9iyvALYazyP1qKjmXFsm6
+OXoRadJcIAJYuGEN27imzt87YQmFciXj63lW4usR7rPpacW004VeWqGfXTnckJd6
+TYFq0xmdhnGxDxOlf6fs5zOEw17NrGlYxQVtdst8sGmpAPMEM7DzvDsjfEPxDuXl
+hQJE8Zk8jK3Xwrnc03NWisZ4QVhgxeR7PVcraFo623qiI/CzH9YqUqMCtIMAqz/T
+COXXl9JxAoGBAOosUC72SM7ZRshneHHszEaZDvfLINdKGUKCDvYlLEmVFqE5iRFy
+SomVci2jtrlGH1gJAWfwkT09JVgtGosRIA0MS82HseLN/QIa01dAmmiZqM/CLbcn
+mpb0CQDkm0Bbz6fokQkFB/sBA5Kj3kOKRydCLp2S0Ugs50cKXDHP5fuVAoGBAMU8
+9rIvmNdweGTiYjHYLJBkzu7eL+5RB2zVSMuZtVivaDTfleilbRlcrBBIaM0urv2W
+UtROB9ack2Ijn/BF+tHkBRVWpaZFdHJ8qMfz2bBDgf6za/LBcvuT35i7ibPT+zfg
+UFXtArmGwPq3AZdWBwIKyN8rM7253WDnUlkN7Ed3AoGBAMPAR0b6meJPvtvHoueZ
+Cyn4yIpbQxi02GjAT8FzUZIxDrm3Xt02rRhV1RxRvm0iMRFmdcZtUvveIVmUWpvl
+tOUzYiptREZT6yvXQNOvLWRDDtqdd5mjgZauaNhWQXGLTgsOXi8sBX/NWS87zJCp
+BtHKgS03jbrHzo2UG32ITLgBAoGAJRoardoWPjCB9ThAkG/BskfERVq2WXYUl3xn
+fSUk39HfIFMOt/ymUScFluqIDFDDyiAE5Lro7o31i3h4FZKUY/conaL29hgKl56r
+gTF1uZp5UZgerkOFhZ2Dag+dD57ImvIvKnqzEIMwufjC69za5J9yucg+q2nTIu9g
+pi/gSnECgYEAhfJ5uq1qa+g23np02ED5ttqmyrMRGGInx3mr2QgJDTum6FujpYCM
+PwJhMwKJZXcf3eUlECSJPa+9UGI53d+JDlQdwq9Pi726KFtrBiS4t9aSyZSpkoWk
+SVdYGaOMtokDKRJibazXjpGFJQy9tAMgtqptS3kL03IuJc643y+lMFc=
+-----END RSA PRIVATE KEY-----
diff --git a/t/plugin/tcp-logger.t b/t/plugin/tcp-logger.t
index 0d15b5692..3ef774f81 100644
--- a/t/plugin/tcp-logger.t
+++ b/t/plugin/tcp-logger.t
@@ -97,7 +97,7 @@ done
"plugins": {
"tcp-logger": {
"host": "127.0.0.1",
- "port": 5044,
+ "port": 3000,
"tls": false
}
},
@@ -134,6 +134,7 @@ hello world
=== TEST 6: error log
+--- log_level: error
--- config
location /t {
content_by_lua_block {
@@ -192,7 +193,7 @@ failed to connect to TCP server: host[312.0.0.1] port[2000]
"plugins": {
"tcp-logger": {
"host": "127.0.0.1",
- "port": 5044,
+ "port": 3000,
"tls": false,
"batch_max_size": 1
}
@@ -226,7 +227,7 @@ failed to connect to TCP server: host[312.0.0.1] port[2000]
"plugins": {
"tcp-logger": {
"host": "127.0.0.1",
- "port": 5045,
+ "port": 43000,
"tls": false,
"batch_max_size": 1
}
@@ -269,8 +270,8 @@ passedopentracing
--- grep_error_log eval
qr/sending a batch logs to 127.0.0.1:(\d+)/
--- grep_error_log_out
-sending a batch logs to 127.0.0.1:5044
-sending a batch logs to 127.0.0.1:5045
+sending a batch logs to 127.0.0.1:3000
+sending a batch logs to 127.0.0.1:43000
@@ -312,7 +313,7 @@ GET /t
"plugins": {
"tcp-logger": {
"host": "127.0.0.1",
- "port": 8125,
+ "port": 3000,
"tls": false,
"batch_max_size": 1,
"inactive_timeout": 1
@@ -338,6 +339,7 @@ GET /t
ngx.HTTP_PUT,
[[{
"log_format": {
+ "case name": "plugin_metadata",
"host": "$host",
"@timestamp": "$time_iso8601",
"client_ip": "$remote_addr"
@@ -349,7 +351,15 @@ GET /t
ngx.say(body)
return
end
- ngx.say(body)
+
+ local code, _, _ = t("/hello", "GET")
+ if code >= 300 then
+ ngx.status = code
+ ngx.say("fail")
+ return
+ end
+
+ ngx.say("passed")
}
}
--- request
@@ -359,47 +369,38 @@ passed
-=== TEST 10: access
---- stream_conf_enable
---- extra_stream_config
- server {
- listen 8125;
- content_by_lua_block {
- local decode = require("toolkit.json").decode
- ngx.log(ngx.WARN, "the mock backend is hit")
+=== TEST 10: log format in plugin_metadata
+--- exec
+tail -n 1 ci/pod/vector/tcp.log
+--- response_body eval
+qr/.*plugin_metadata.*/
- local sock, err = ngx.req.socket(true)
- if not sock then
- ngx.log(ngx.ERR, "failed to get the request socket: ", err)
- return
- end
- local data, err = sock:receive('*a')
- if not data then
- if err and err ~= "closed" then
- ngx.log(ngx.ERR, "socket error, returning: ", err)
- end
- return
- end
+=== TEST 11: remove tcp logger metadata
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+
+ local code, body = t('/apisix/admin/plugin_metadata/tcp-logger',
+ ngx.HTTP_PUT,
+ [[{
+ "log_format": {}
+ }]]
+ )
- data = decode(data)
- assert(data.client_ip == "127.0.0.1")
+ ngx.say(body)
}
}
--- request
-GET /hello
+GET /t
--- response_body
-hello world
---- wait: 2
---- error_log
-the mock backend is hit
---- no_error_log
-[error]
+passed
-=== TEST 11: log format in plugin
+=== TEST 12: log format in plugin
--- config
location /t {
content_by_lua_block {
@@ -410,9 +411,10 @@ the mock backend is hit
"plugins": {
"tcp-logger": {
"host": "127.0.0.1",
- "port": 8125,
+ "port": 3000,
"tls": false,
"log_format": {
+ "case name": "logger format in plugin",
"vip": "$remote_addr"
},
"batch_max_size": 1,
@@ -435,50 +437,84 @@ the mock backend is hit
return
end
- ngx.say(body)
+ local code, _, body2 = t("/hello", "GET")
+ if code >= 300 then
+ ngx.status = code
+ ngx.say("fail")
+ return
+ end
+
+ ngx.say("passed")
}
}
--- request
GET /t
+--- wait: 0.5
--- response_body
passed
-=== TEST 12: access
---- stream_conf_enable
---- extra_stream_config
- server {
- listen 8125;
+=== TEST 13: check tcp log
+--- exec
+tail -n 1 ci/pod/vector/tcp.log
+--- response_body eval
+qr/.*logger format in plugin.*/
+
+
+
+=== TEST 14: true tcp log with tls
+--- config
+ location /t {
content_by_lua_block {
- local decode = require("toolkit.json").decode
- ngx.log(ngx.WARN, "the mock backend is hit")
+ local t = require("lib.test_admin").test
+ local code, body1 = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "tcp-logger": {
+ "host": "127.0.0.1",
+ "port": 43000,
+ "tls": true,
+ "batch_max_size": 1
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1982": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/opentracing"
+ }]]
+ )
- local sock, err = ngx.req.socket(true)
- if not sock then
- ngx.log(ngx.ERR, "failed to get the request socket: ", err)
+ if code >= 300 then
+ ngx.status = code
+ ngx.say("fail")
return
end
- local data, err = sock:receive('*a')
-
- if not data then
- if err and err ~= "closed" then
- ngx.log(ngx.ERR, "socket error, returning: ", err)
- end
+ local code, _, body2 = t("/opentracing", "GET")
+ if code >= 300 then
+ ngx.status = code
+ ngx.say("fail")
return
end
- data = decode(data)
- assert(data.vip == "127.0.0.1")
+ ngx.print(body2)
}
}
--- request
-GET /hello
+GET /t
+--- wait: 0.5
--- response_body
-hello world
---- wait: 2
---- error_log
-the mock backend is hit
---- no_error_log
-[error]
+opentracing
+
+
+
+=== TEST 15: check tls log
+--- exec
+tail -n 1 ci/pod/vector/tls-datas.log
+--- response_body eval
+qr/.*route_id.*1.*/
