This is an automated email from the ASF dual-hosted git repository. spacewander pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push: new a47dab7 feat: improve kubernetes discovery (#6663) a47dab7 is described below commit a47dab7eecd90998499eae10b477ce7fd8175c28 Author: zhixiongdu <r...@libssl.com> AuthorDate: Mon Mar 21 08:46:26 2022 +0800 feat: improve kubernetes discovery (#6663) --- apisix/cli/ngx_tpl.lua | 4 + apisix/cli/ops.lua | 6 + apisix/discovery/kubernetes/informer_factory.lua | 4 +- apisix/discovery/kubernetes/init.lua | 15 +- conf/config-default.yaml | 1 + docs/en/latest/config.json | 3 +- docs/en/latest/discovery/kubernetes.md | 188 +++++++++++++++++++++++ docs/zh/latest/discovery/kubernetes.md | 106 ++++++++++--- t/APISIX.pm | 1 + 9 files changed, 296 insertions(+), 32 deletions(-) diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index 4f1c8c2..b1ec2bb 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -190,6 +190,10 @@ http { lua_shared_dict balancer-ewma-last-touched-at {* http.lua_shared_dict["balancer-ewma-last-touched-at"] *}; lua_shared_dict etcd-cluster-health-check {* http.lua_shared_dict["etcd-cluster-health-check"] *}; # etcd health check + {% if enabled_discoveries["kubernetes"] then %} + lua_shared_dict kubernetes {* http.lua_shared_dict["kubernetes"] *}; + {% end %} + {% if enabled_plugins["limit-conn"] then %} lua_shared_dict plugin-limit-conn {* http.lua_shared_dict["plugin-limit-conn"] *}; {% end %} diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua index 7208e5c..2844126 100644 --- a/apisix/cli/ops.lua +++ b/apisix/cli/ops.lua @@ -251,6 +251,11 @@ Please modify "admin_key" in conf/config.yaml . use_apisix_openresty = false end + local enabled_discoveries = {} + for name in pairs(yaml_conf.discovery or {}) do + enabled_discoveries[name] = true + end + local enabled_plugins = {} for i, name in ipairs(yaml_conf.plugins or {}) do enabled_plugins[name] = true @@ -528,6 +533,7 @@ Please modify "admin_key" in conf/config.yaml . with_module_status = with_module_status, use_apisix_openresty = use_apisix_openresty, error_log = {level = "warn"}, + enabled_discoveries = enabled_discoveries, enabled_plugins = enabled_plugins, enabled_stream_plugins = enabled_stream_plugins, dubbo_upstream_multiplex_count = dubbo_upstream_multiplex_count, diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua index 8b50fc3..a03f27a 100644 --- a/apisix/discovery/kubernetes/informer_factory.lua +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -23,8 +23,6 @@ local type = type local core = require("apisix.core") local http = require("resty.http") -local empty_table = {} - local function list_query(informer) local arguments = { limit = informer.limit, @@ -81,7 +79,7 @@ local function list(httpc, apiserver, informer) informer.version = data.metadata.resourceVersion if informer.on_added then - for _, item in ipairs(data.items or empty_table) do + for _, item in ipairs(data.items or {}) do informer:on_added(item, "list") end end diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index ba83588..a0491be 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -31,6 +31,7 @@ local local_conf = require("apisix.core.config_local").local_conf() local informer_factory = require("apisix.discovery.kubernetes.informer_factory") local endpoint_dict + local default_weight local endpoint_lrucache = core.lrucache.new({ @@ -39,7 +40,6 @@ local endpoint_lrucache = core.lrucache.new({ }) local endpoint_buffer = {} -local empty_table = {} local function sort_nodes_cmp(left, right) if left.host ~= right.host then @@ -60,10 +60,10 @@ local function on_endpoint_modified(informer, endpoint) core.table.clear(endpoint_buffer) local subsets = endpoint.subsets - for _, subset in ipairs(subsets or empty_table) do + for _, subset in ipairs(subsets or {}) do if subset.addresses then local addresses = subset.addresses - for _, port in ipairs(subset.ports or empty_table) do + for _, port in ipairs(subset.ports or {}) do local port_name if port.name then port_name = port.name @@ -166,7 +166,7 @@ local function setup_namespace_selector(conf, informer) local match = conf.namespace_selector.match local m, err for _, v in ipairs(match) do - m, err = ngx.re.match(namespace, v, "j") + m, err = ngx.re.match(namespace, v, "jo") if m and m[0] == namespace then return true end @@ -324,10 +324,9 @@ end function _M.init_worker() - -- TODO: maybe we can read dict name from discovery config - endpoint_dict = ngx.shared.discovery + endpoint_dict = ngx.shared.kubernetes if not endpoint_dict then - error("failed to get nginx shared dict: discovery, please check your APISIX version") + error("failed to get lua_shared_dict: kubernetes, please check your APISIX version") end if process.type() ~= "privileged agent" then @@ -336,7 +335,7 @@ function _M.init_worker() local discovery_conf = local_conf.discovery.kubernetes - default_weight = discovery_conf.default_weight or 50 + default_weight = discovery_conf.default_weight local apiserver, err = get_apiserver(discovery_conf) if err then diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 450b6f1..d42ff21 100644 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -264,6 +264,7 @@ nginx_config: # config for render the template to generate n introspection: 10m access-tokens: 1m ext-plugin: 1m + kubernetes: 1m etcd: host: # it's possible to define multiple etcd hosts addresses of the same etcd cluster. diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json index 0bf31b8..5c03681 100644 --- a/docs/en/latest/config.json +++ b/docs/en/latest/config.json @@ -206,7 +206,8 @@ "discovery/dns", "discovery/consul_kv", "discovery/nacos", - "discovery/eureka" + "discovery/eureka", + "discovery/kubernetes" ] }, { diff --git a/docs/en/latest/discovery/kubernetes.md b/docs/en/latest/discovery/kubernetes.md new file mode 100644 index 0000000..0bf7431 --- /dev/null +++ b/docs/en/latest/discovery/kubernetes.md @@ -0,0 +1,188 @@ +--- +title: Kubernetes +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +## Summary + +The [_Kubernetes_](https://kubernetes.io/) service discovery [_List-Watch_](https://kubernetes.io/docs/reference/using-api/api-concepts/) real-time changes of [_Endpoints_](https://kubernetes.io/docs/concepts/services-networking/service/) resources, +then store theirs value into ngx.shared.kubernetes \ +Discovery also provides a query interface in accordance with the [_APISIX Discovery Specification_](https://github.com/apache/apisix/blob/master/docs/en/latest/discovery.md) + +## Configuration + +A detailed configuration for the kubernetes service discovery is as follows: + +```yaml +discovery: + kubernetes: + service: + # apiserver schema, options [http, https] + schema: https #default https + + # apiserver host, options [ipv4, ipv6, domain, environment variable] + host: ${KUBERNETES_SERVICE_HOST} #default ${KUBERNETES_SERVICE_HOST} + + # apiserver port, options [port number, environment variable] + port: ${KUBERNETES_SERVICE_PORT} #default ${KUBERNETES_SERVICE_PORT} + + client: + # serviceaccount token or token_file + token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + + #token: |- + # eyJhbGciOiJSUzI1NiIsImtpZCI6Ikx5ME1DNWdnbmhQNkZCNlZYMXBsT3pYU3BBS2swYzBPSkN3ZnBESGpkUEEif + # 6Ikx5ME1DNWdnbmhQNkZCNlZYMXBsT3pYU3BBS2swYzBPSkN3ZnBESGpkUEEifeyJhbGciOiJSUzI1NiIsImtpZCI + + # kubernetes discovery plugin support use namespace_selector + # you can use one of [equal, not_equal, match, not_match] filter namespace + namespace_selector: + # only save endpoints with namespace equal default + equal: default + + # only save endpoints with namespace not equal default + #not_equal: default + + # only save endpoints with namespace match one of [default, ^my-[a-z]+$] + #match: + #- default + #- ^my-[a-z]+$ + + # only save endpoints with namespace not match one of [default, ^my-[a-z]+$ ] + #not_match: + #- default + #- ^my-[a-z]+$ + + # kubernetes discovery plugin support use label_selector + # for the expression of label_selector, please refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + label_selector: |- + first="a",second="b" +``` + +If the kubernetes service discovery runs inside a pod, you can use minimal configuration: + +```yaml +discovery: + kubernetes: { } +``` + +If the kubernetes service discovery runs outside a pod, you need to create or select a specified [_ServiceAccount_](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/), +then get its token value, and use following configuration: + +```yaml +discovery: + kubernetes: + service: + schema: https + host: # enter apiserver host value here + port: # enter apiserver port value here + client: + token: # enter serviceaccount token value here + #token_file: # enter file path here +``` + +## Interface + +the kubernetes service discovery provides a query interface in accordance with the [_APISIX Discovery Specification_](https://github.com/apache/apisix/blob/master/docs/en/latest/discovery.md) + +**function:** \ + nodes(service_name) + +**description:** \ + nodes() function attempts to look up the ngx.shared.kubernetes for nodes corresponding to service_name, \ + service_name should match pattern: _[namespace]/[name]:[portName]_ + + + namespace: The namespace where the kubernetes endpoints is located + + + name: The name of the kubernetes endpoints + + + portName: The portName of the kubernetes endpoints, if there is no portName, use targetPort, port instead + +**return value:** \ + if the kubernetes endpoints value is as follows: + + ```yaml + apiVersion: v1 + kind: Endpoints + metadata: + name: plat-dev + namespace: default + subsets: + - addresses: + - ip: "10.5.10.109" + - ip: "10.5.10.110" + ports: + - port: 3306 + ``` + + a nodes("default/plat-dev:3306") call will get follow result: + + ``` + { + { + host="10.5.10.109", + port= 3306, + weight= 50, + }, + { + host="10.5.10.110", + port= 3306, + weight= 50, + }, + } + ``` + +## Q&A + +> Q: Why only support configuration token to access _Kubernetes APIServer_ \ +> A: Usually, we will use three ways to complete the authentication of _Kubernetes APIServer_: +> +>+ mTLS +>+ token +>+ basic authentication +> +> Because lua-resty-http does not currently support mTLS, and basic authentication is not recommended,\ +> So currently only the token authentication method is implemented + +--- + +> Q: APISIX inherits Nginx's multiple process model, does it mean that each nginx worker process will [_List-Watch_](https://kubernetes.io/docs/reference/using-api/api-concepts/) kubernetes endpoints resources \ +> A: The kubernetes service discovery only uses privileged processes to [_List-Watch_](https://kubernetes.io/docs/reference/using-api/api-concepts/) kubernetes endpoints resources, then store theirs value \ +> into ngx.shared.kubernetes, worker processes get results by querying ngx.shared.kubernetes + +--- + +> Q: How to get [_ServiceAccount_](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) token value \ +> A: Assume your [_ServiceAccount_](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) located in namespace apisix and name is kubernetes-discovery, you can use the following steps to get token value +> +> 1. Get secret name: \ +> you can execute the following command, the output of the first column is the secret name we want +> +> ```shell +> kubectl -n apisix get secrets | grep kubernetes-discovery +> ``` +> +> 2. Get token value: \ +> assume secret resources name is kubernetes-discovery-token-c64cv, you can execute the following command, the output is the service account token value we want +> +> ```shell +> kubectl -n apisix get secret kubernetes-discovery-token-c64cv -o jsonpath={.data.token} | base64 -d +> ``` diff --git a/docs/zh/latest/discovery/kubernetes.md b/docs/zh/latest/discovery/kubernetes.md index 9ef14ff..3185fed 100644 --- a/docs/zh/latest/discovery/kubernetes.md +++ b/docs/zh/latest/discovery/kubernetes.md @@ -1,3 +1,7 @@ +--- +title: Kubernetes +--- + <!-- # # Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,14 +21,15 @@ # --> -# 基于 Kubernetes 的服务发现 +## 基于 Kubernetes 的服务发现 -Kubernetes 服务发现插件以 ListWatch 方式监听 Kubernetes 集群 v1.endpoints 的实时变化, -并将其值存储在 ngx.shared.dict 中, 同时遵循 APISIX Discovery 规范提供查询接口 +Kubernetes 服务发现模块以 [_List-Watch_](https://kubernetes.io/docs/reference/using-api/api-concepts) 方式监听 [_Kubernetes_](https://kubernetes.io) 集群 [_Endpoints_](https://kubernetes.io/docs/concepts/services-networking/service) 资源的实时变化, +并将其值存储到 ngx.shared.kubernetes 中 \ +模块同时遵循 [_APISIX Discovery 规范_](https://github.com/apache/apisix/blob/master/docs/zh/latest/discovery.md) 提供了节点查询接口 -# Kubernetes 服务发现插件的配置 +## Kubernetes 服务发现模块的配置 -Kubernetes 服务发现插件的样例配置如下: +Kubernetes 服务发现模块的完整配置如下: ```yaml discovery: @@ -72,14 +77,14 @@ discovery: first="a",second="b" ``` -如果 Kubernetes 服务插件运行在 Pod 内, 你可以使用最简配置: +如果 Kubernetes 服务发现模块运行在 Pod 内, 你可以使用最简配置: ```yaml discovery: kubernetes: { } ``` -如果 Kubernetes 服务插件运行在 Pod 外, 你需要新建或选取指定的 ServiceAccount, 获取其 Token 值, 并使用如下配置: +如果 Kubernetes 服务发现模块运行在 Pod 外, 你需要新建或选取指定的 [_ServiceAccount_](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/), 获取其 Token 值, 然后使用如下配置: ```yaml discovery: @@ -93,28 +98,89 @@ discovery: #token_file: # enter file path here ``` -# Kubernetes 服务发现插件的使用 +## Kubernetes 服务发现模块的查询接口 + +Kubernetes 服务发现模块遵循 [_APISIX Discovery 规范_](https://github.com/apache/apisix/blob/master/docs/zh/latest/discovery.md) 提供查询接口 + +**函数:** + nodes(service_name) + +**说明:** + service_name 必须满足格式: [namespace]/[name]:[portName] + + + namespace: Endpoints 所在的命名空间 + + + name: Endpoints 的资源名 + + + portName: Endpoints 定义包含的 portName, 如果 Endpoints 没有定义 portName, 请使用 targetPort,Port 代替 + +**返回值:** + 以如下 Endpoints 为例: -Kubernetes 服务发现插件提供与其他服务发现插件相同的查询接口 -> nodes(service_name) \ -service_name 的 pattern 如下: -> _[namespace]/[name]:[portName]_ + ```yaml + apiVersion: v1 + kind: Endpoints + metadata: + name: plat-dev + namespace: default + subsets: + - addresses: + - ip: "10.5.10.109" + - ip: "10.5.10.110" + ports: + - port: 3306 + ``` -如果 kubernetes Endpoint 没有定义 portName, Kubernetes 服务发现插件会依次使用 targetPort, port 代替 + nodes("default/plat-dev:3306") 调用会得到如下的返回值: -# Q&A + ``` + { + { + host="10.5.10.109", + port= 3306, + weight= 50, + }, + { + host="10.5.10.110", + port= 3306, + weight= 50, + }, + } + ``` -> Q: 为什么只支持配置 token 来访问 Kubernetes ApiServer \ -> A: 通常情况下,我们会使用三种方式与 Kubernetes ApiServer 通信 : +## Q&A + +> Q: 为什么只支持配置 token 来访问 Kubernetes APIServer \ +> A: 一般情况下,我们有三种方式可以完成与 Kubernetes APIServer 的认证: > >+ mTLS >+ token >+ basic authentication > -> 因为 lua-resty-http 目前不支持 mTLS, 以及 basic authentication 不被推荐使用,\ +> 因为 lua-resty-http 目前不支持 mTLS, basic authentication 不被推荐使用,\ > 所以当前只实现了 token 认证方式 -------- +--- + +> Q: APISIX 继承了 Nginx 的多进程模型, 是否意味着每个 APISIX 工作进程都会监听 Kubernetes Endpoints \ +> A: Kubernetes 服务发现模块只使用特权进程监听 Kubernetes Endpoints, 然后将其值存储\ +> 到 ngx.shared.kubernetes, 工作进程通过查询 ngx.shared.kubernetes 来获取结果 -> Q: APISIX 是多进程模型, 是否意味着每个 APISIX 工作进程都会监听 Kubernetes v1.endpoints \ -> A: Kubernetes 服务发现插件只使用特权进程监听 Kubernetes v1.endpoints, 然后将结果存储\ -> 在 ngx.shared.dict 中, 业务进程是通过查询 ngx.shared.dict 来获取结果的 +--- + +> Q: 怎样获取指定 ServiceAccount 的 Token 值 \ +> A: 假定你指定的 ServiceAccount 资源名为 “kubernetes-discovery“, 命名空间为 “apisix”, 请按如下步骤获取其 Token 值 +> +> 1. 获取 _Secret_ 资源名: \ +> 执行以下命令, 输出的第一列内容就是目标 _Secret_ 资源名 +> +> ```shell +> kubectl -n apisix get secrets | grep kubernetes-discovery +> ``` +> +> 2. 获取 Token 值: \ +> 假定你获取到的 _Secret_ 资源名为 "kubernetes-discovery-token-c64cv", 执行以下命令, 输出内容就是目标 Token 值 +> +> ```shell +> kubectl -n apisix get secret kubernetes-discovery-token-c64cv -o jsonpath={.data.token} | base64 -d +> ``` diff --git a/t/APISIX.pm b/t/APISIX.pm index 671ffe3..b9f7089 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -507,6 +507,7 @@ _EOC_ lua_capture_error_log 1m; # plugin error-log-logger lua_shared_dict etcd-cluster-health-check 10m; # etcd health check lua_shared_dict ext-plugin 1m; + lua_shared_dict kubernetes 1m; proxy_ssl_name \$upstream_host; proxy_ssl_server_name on;