AlinsRan commented on code in PR #13509: URL: https://github.com/apache/apisix/pull/13509#discussion_r3549401535
########## apisix/secret/kubernetes.lua: ########## @@ -0,0 +1,262 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +--- Kubernetes Secret Manager. +-- Fetches secrets from the Kubernetes API server using the pod's ServiceAccount +-- token. This allows APISIX to read Kubernetes Secrets directly from the cluster +-- it is running in, without requiring an external secrets management service. +-- +-- URI format: +-- $secret://kubernetes/{manager-id}/{namespace}/{secret-name}/{data-key} +-- +-- Example: +-- $secret://kubernetes/my-k8s/default/my-secret/password +-- +-- The manager is configured once via the Admin API: +-- PUT /apisix/admin/secrets/kubernetes/my-k8s +-- { +-- "service_account_file": "/var/run/secrets/kubernetes.io/serviceaccount/token", +-- "kubernetes_host": "kubernetes.default.svc", +-- "kubernetes_port": "443", +-- "ssl_verify": true +-- } +-- +-- All fields have sensible defaults for in-cluster usage, so the minimal +-- valid configuration is an empty object `{}`. +-- +-- TLS note: when ssl_verify is true (the default), the TLS handshake is +-- validated against the nginx-level lua_ssl_trusted_certificate bundle. +-- For in-cluster usage, set apisix.ssl.ssl_trusted_certificate in config.yaml +-- to /var/run/secrets/kubernetes.io/serviceaccount/ca.crt so that the +-- kube-apiserver certificate (signed by the cluster CA) is trusted. + +local core = require("apisix.core") +local http = require("resty.http") +local env = core.env + +local io_open = io.open +local find = core.string.find +local sub = core.string.sub +local ngx_decode_base64 = ngx.decode_base64 + +local DEFAULT_SA_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/token" + +local schema = { + type = "object", + properties = { + service_account_file = { + type = "string", + description = "Path to the ServiceAccount token file. " + .. "Defaults to the standard in-cluster path.", + default = DEFAULT_SA_FILE, + }, + kubernetes_host = { + type = "string", + description = "Kubernetes API server hostname or IP. " + .. "Defaults to the KUBERNETES_SERVICE_HOST environment variable.", + }, + kubernetes_port = { + type = "string", + description = "Kubernetes API server port. " + .. "Defaults to the KUBERNETES_SERVICE_PORT environment variable.", + }, + endpoint = { + type = "string", + description = "Full base URL of the Kubernetes API server " + .. "(e.g. https://kubernetes.default.svc:443). " + .. "When set, kubernetes_host and kubernetes_port are ignored. " + .. "Useful for testing with plain HTTP mock servers.", + }, + ssl_verify = { + type = "boolean", + description = "Verify the Kubernetes API server TLS certificate. " + .. "When true, validation uses the nginx-level " + .. "lua_ssl_trusted_certificate bundle; set " + .. "apisix.ssl.ssl_trusted_certificate to the cluster CA " + .. "(/var/run/secrets/kubernetes.io/serviceaccount/ca.crt) " + .. "in config.yaml for in-cluster usage.", + default = true, + }, + }, + required = {}, +} + +local _M = { + schema = schema, +} + + +local function read_file(path) + local f, err = io_open(path, "r") + if not f then + return nil, "failed to open file " .. path .. ": " .. err + end + local content = f:read("*a") + f:close() + if not content or content == "" then + return nil, "file is empty: " .. path + end + return content +end + + +local function make_request_to_k8s(conf, namespace, secret_name) + local sa_file = conf.service_account_file or DEFAULT_SA_FILE + local token, err = read_file(sa_file) + if not token then + return nil, err + end + -- strip trailing newline + token = token:gsub("%s+$", "") + + local k8s_host = conf.kubernetes_host + or env.fetch_by_uri("$ENV://KUBERNETES_SERVICE_HOST") + or os.getenv("KUBERNETES_SERVICE_HOST") Review Comment: This is the lint failure: `ERROR: apisix/secret/kubernetes.lua: line 128: getting the Lua global "os"` (same on line 132). Use `core.env.get("KUBERNETES_SERVICE_HOST")` instead. `apisix/discovery/kubernetes/core.lua:74` already made this switch — the comment there explains it works around a lua-resty-core prefix-collision bug with `os.getenv` during worker init (#13055). The fallback is also redundant: `env.fetch_by_uri("$ENV://...")` on the line above already goes through `core.env`'s `_M.get`, which checks the environ snapshot and then falls back to `os.getenv` itself. So `core.env.get(...)` alone covers both branches. ########## apisix/secret/kubernetes.lua: ########## @@ -0,0 +1,262 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +--- Kubernetes Secret Manager. +-- Fetches secrets from the Kubernetes API server using the pod's ServiceAccount +-- token. This allows APISIX to read Kubernetes Secrets directly from the cluster +-- it is running in, without requiring an external secrets management service. +-- +-- URI format: +-- $secret://kubernetes/{manager-id}/{namespace}/{secret-name}/{data-key} +-- +-- Example: +-- $secret://kubernetes/my-k8s/default/my-secret/password +-- +-- The manager is configured once via the Admin API: +-- PUT /apisix/admin/secrets/kubernetes/my-k8s +-- { +-- "service_account_file": "/var/run/secrets/kubernetes.io/serviceaccount/token", +-- "kubernetes_host": "kubernetes.default.svc", +-- "kubernetes_port": "443", +-- "ssl_verify": true +-- } +-- +-- All fields have sensible defaults for in-cluster usage, so the minimal +-- valid configuration is an empty object `{}`. +-- +-- TLS note: when ssl_verify is true (the default), the TLS handshake is +-- validated against the nginx-level lua_ssl_trusted_certificate bundle. +-- For in-cluster usage, set apisix.ssl.ssl_trusted_certificate in config.yaml +-- to /var/run/secrets/kubernetes.io/serviceaccount/ca.crt so that the +-- kube-apiserver certificate (signed by the cluster CA) is trusted. + +local core = require("apisix.core") +local http = require("resty.http") +local env = core.env + +local io_open = io.open +local find = core.string.find +local sub = core.string.sub +local ngx_decode_base64 = ngx.decode_base64 + +local DEFAULT_SA_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/token" + +local schema = { + type = "object", + properties = { + service_account_file = { + type = "string", + description = "Path to the ServiceAccount token file. " + .. "Defaults to the standard in-cluster path.", + default = DEFAULT_SA_FILE, + }, + kubernetes_host = { + type = "string", + description = "Kubernetes API server hostname or IP. " + .. "Defaults to the KUBERNETES_SERVICE_HOST environment variable.", + }, + kubernetes_port = { + type = "string", + description = "Kubernetes API server port. " + .. "Defaults to the KUBERNETES_SERVICE_PORT environment variable.", + }, + endpoint = { + type = "string", + description = "Full base URL of the Kubernetes API server " + .. "(e.g. https://kubernetes.default.svc:443). " + .. "When set, kubernetes_host and kubernetes_port are ignored. " + .. "Useful for testing with plain HTTP mock servers.", + }, + ssl_verify = { + type = "boolean", + description = "Verify the Kubernetes API server TLS certificate. " + .. "When true, validation uses the nginx-level " + .. "lua_ssl_trusted_certificate bundle; set " + .. "apisix.ssl.ssl_trusted_certificate to the cluster CA " + .. "(/var/run/secrets/kubernetes.io/serviceaccount/ca.crt) " + .. "in config.yaml for in-cluster usage.", + default = true, + }, + }, + required = {}, +} + +local _M = { + schema = schema, +} + + +local function read_file(path) + local f, err = io_open(path, "r") + if not f then + return nil, "failed to open file " .. path .. ": " .. err + end + local content = f:read("*a") + f:close() + if not content or content == "" then + return nil, "file is empty: " .. path + end + return content +end + + +local function make_request_to_k8s(conf, namespace, secret_name) + local sa_file = conf.service_account_file or DEFAULT_SA_FILE + local token, err = read_file(sa_file) + if not token then + return nil, err + end + -- strip trailing newline + token = token:gsub("%s+$", "") + + local k8s_host = conf.kubernetes_host Review Comment: Heads up on the in-cluster "just works" path: nginx workers only see env vars declared with an `env` directive. `apisix/cli/ngx_tpl.lua` emits `APISIX_PROFILE`, `PATH`, `GCP_SERVICE_ACCOUNT` etc., but not `KUBERNETES_SERVICE_HOST`/`KUBERNETES_SERVICE_PORT`. So with the documented minimal config (`{}`) this resolves to nil and you get `kubernetes_host is not set and KUBERNETES_SERVICE_HOST env var is missing`. Either document that users must declare both under `nginx_config.envs` in config.yaml, or drop the env fallback and require `kubernetes_host`. Right now the doc table presents `$KUBERNETES_SERVICE_HOST` as a working default. Worth noting this branch has no test coverage at all — every test passes an explicit `endpoint`. ########## apisix/secret/kubernetes.lua: ########## @@ -0,0 +1,262 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +--- Kubernetes Secret Manager. +-- Fetches secrets from the Kubernetes API server using the pod's ServiceAccount +-- token. This allows APISIX to read Kubernetes Secrets directly from the cluster +-- it is running in, without requiring an external secrets management service. +-- +-- URI format: +-- $secret://kubernetes/{manager-id}/{namespace}/{secret-name}/{data-key} +-- +-- Example: +-- $secret://kubernetes/my-k8s/default/my-secret/password +-- +-- The manager is configured once via the Admin API: +-- PUT /apisix/admin/secrets/kubernetes/my-k8s +-- { +-- "service_account_file": "/var/run/secrets/kubernetes.io/serviceaccount/token", +-- "kubernetes_host": "kubernetes.default.svc", +-- "kubernetes_port": "443", +-- "ssl_verify": true +-- } +-- +-- All fields have sensible defaults for in-cluster usage, so the minimal +-- valid configuration is an empty object `{}`. +-- +-- TLS note: when ssl_verify is true (the default), the TLS handshake is +-- validated against the nginx-level lua_ssl_trusted_certificate bundle. +-- For in-cluster usage, set apisix.ssl.ssl_trusted_certificate in config.yaml +-- to /var/run/secrets/kubernetes.io/serviceaccount/ca.crt so that the +-- kube-apiserver certificate (signed by the cluster CA) is trusted. + +local core = require("apisix.core") +local http = require("resty.http") +local env = core.env + +local io_open = io.open +local find = core.string.find +local sub = core.string.sub +local ngx_decode_base64 = ngx.decode_base64 + +local DEFAULT_SA_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/token" + +local schema = { + type = "object", + properties = { + service_account_file = { + type = "string", + description = "Path to the ServiceAccount token file. " + .. "Defaults to the standard in-cluster path.", + default = DEFAULT_SA_FILE, + }, + kubernetes_host = { + type = "string", + description = "Kubernetes API server hostname or IP. " + .. "Defaults to the KUBERNETES_SERVICE_HOST environment variable.", + }, + kubernetes_port = { + type = "string", + description = "Kubernetes API server port. " + .. "Defaults to the KUBERNETES_SERVICE_PORT environment variable.", + }, + endpoint = { + type = "string", + description = "Full base URL of the Kubernetes API server " + .. "(e.g. https://kubernetes.default.svc:443). " + .. "When set, kubernetes_host and kubernetes_port are ignored. " + .. "Useful for testing with plain HTTP mock servers.", + }, + ssl_verify = { + type = "boolean", + description = "Verify the Kubernetes API server TLS certificate. " + .. "When true, validation uses the nginx-level " + .. "lua_ssl_trusted_certificate bundle; set " + .. "apisix.ssl.ssl_trusted_certificate to the cluster CA " + .. "(/var/run/secrets/kubernetes.io/serviceaccount/ca.crt) " + .. "in config.yaml for in-cluster usage.", + default = true, + }, + }, + required = {}, Review Comment: This line makes the whole manager unusable. An empty Lua table serializes to an empty JSON **object**, not an array, so `jsonschema` reads it as "the input must be an empty table" and rejects any config that has properties: ``` required={} + {} -> OK required={} + {ssl_verify=true} -> FAIL: the input data should be an empty table no required + {ssl_verify=true} -> OK ``` Both `apisix/secret.lua:52` (`check_secret`) and `apisix/admin/secrets.lua:35` run `core.schema.check(secret_manager.schema, conf)`, so `PUT /apisix/admin/secrets/kubernetes/{id}` is rejected for every non-empty body. That's why TEST 13 returns `{"error_msg":"invalid configuration: the input data should be an empty table"}` and TEST 14 then cascades to `no secret conf`. Just delete the line — `vault.lua`, `aws.lua` and `gcp.lua` all omit `required` when nothing is mandatory. ########## docs/en/latest/terminology/secret.md: ########## @@ -363,3 +364,130 @@ curl http://127.0.0.1:9180/apisix/admin/secrets/gcp/1 \ }' ``` + +## Use Kubernetes Secrets to manage secrets + +When APISIX is running inside a Kubernetes cluster, it can read secrets directly +from the Kubernetes API server using the pod's ServiceAccount credentials. This +allows you to manage APISIX plugin credentials through standard Kubernetes Secrets +without requiring an external secrets management service. + +### Prerequisites + +The ServiceAccount used by the APISIX pod must have RBAC permissions to read +the target Secrets. Create a `ClusterRole` and bind it to the APISIX ServiceAccount: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: apisix-secret-reader +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: apisix-secret-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: apisix-secret-reader +subjects: + - kind: ServiceAccount + name: apisix + namespace: apisix +``` + +### Usage + +``` +$secret://kubernetes/{manager-id}/{namespace}/{secret-name}/{data-key} +``` + +- `manager-id`: the ID of the Kubernetes secret manager instance registered via Admin API +- `namespace`: the Kubernetes namespace where the Secret lives +- `secret-name`: the name of the Kubernetes Secret +- `data-key`: the key within `Secret.data` (the value will be base64-decoded automatically) + +### Configuration via Admin API + +Register a Kubernetes secret manager instance. All fields are optional and default +to standard in-cluster values: + +```bash +curl -X PUT http://127.0.0.1:9180/apisix/admin/secrets/kubernetes/my-k8s \ + -H "X-API-KEY: $ADMIN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "service_account_file": "/var/run/secrets/kubernetes.io/serviceaccount/token", + "kubernetes_host": "kubernetes.default.svc", + "kubernetes_port": "443", + "ssl_verify": true + }' +``` + +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `service_account_file` | string | No | `/var/run/secrets/kubernetes.io/serviceaccount/token` | Path to the ServiceAccount token file | +| `kubernetes_host` | string | No | `$KUBERNETES_SERVICE_HOST` env var | Kubernetes API server hostname or IP | +| `kubernetes_port` | string | No | `$KUBERNETES_SERVICE_PORT` env var | Kubernetes API server port | +| `endpoint` | string | No | derived from `kubernetes_host`/`kubernetes_port` | Full base URL of the API server (e.g. `https://kubernetes.default.svc:443`). Overrides `kubernetes_host` and `kubernetes_port` when set | +| `ssl_verify` | boolean | No | `true` | Verify the Kubernetes API server TLS certificate | + +#### TLS configuration for in-cluster usage + +When `ssl_verify` is `true` (the default), TLS verification uses the nginx-level +`lua_ssl_trusted_certificate` bundle. The cluster CA that signs the kube-apiserver +certificate is **not** in the system CA bundle, so you must add it explicitly in +`config.yaml`: + +```yaml +apisix: + ssl: + ssl_trusted_certificate: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt +``` + +Without this setting the TLS handshake will fail. Alternatively, set `ssl_verify: false`, +but this is not recommended in production because the ServiceAccount token is sent +to an unverified endpoint. + +### Example: protect plugin credentials with Kubernetes Secrets + +Suppose you have a Kubernetes Secret in the `my-app` namespace: + +```bash +kubectl create secret generic keycloak-creds \ + --namespace my-app \ + --from-literal=client_id=my-client-id \ + --from-literal=client_secret=my-client-secret +``` + +You can reference these values in the `authz-keycloak` plugin configuration: + +```bash +curl -X PUT http://127.0.0.1:9180/apisix/admin/routes/1 \ + -H "X-API-KEY: $ADMIN_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "uri": "/api/*", + "plugins": { + "authz-keycloak": { + "discovery": "https://keycloak.example.com/auth/realms/my-realm/.well-known/uma2-configuration";, + "client_id": "$secret://kubernetes/my-k8s/my-app/keycloak-creds/client_id", + "client_secret": "$secret://kubernetes/my-k8s/my-app/keycloak-creds/client_secret", + "policy_enforcement_mode": "ENFORCING" + } + }, + "upstream": { + "type": "roundrobin", + "nodes": {"backend-service:8080": 1} + } + }' +``` + +APISIX resolves `$secret://kubernetes/...` references at request time by calling Review Comment: This overstates it. `apisix/plugin.lua:113` calls `secret.fetch_secrets(conf, true)` with `use_cache=true`, and `apisix/secret.lua` caches resolved values in an lrucache with a 300s TTL (60s neg_ttl). So most requests never reach the Kubernetes API, and a rotated secret stays stale for up to ~5 minutes. Worth saying that explicitly, since "picked up automatically" reads as immediate. ########## apisix/secret/kubernetes.lua: ########## @@ -0,0 +1,262 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +--- Kubernetes Secret Manager. +-- Fetches secrets from the Kubernetes API server using the pod's ServiceAccount +-- token. This allows APISIX to read Kubernetes Secrets directly from the cluster +-- it is running in, without requiring an external secrets management service. +-- +-- URI format: +-- $secret://kubernetes/{manager-id}/{namespace}/{secret-name}/{data-key} +-- +-- Example: +-- $secret://kubernetes/my-k8s/default/my-secret/password +-- +-- The manager is configured once via the Admin API: +-- PUT /apisix/admin/secrets/kubernetes/my-k8s +-- { +-- "service_account_file": "/var/run/secrets/kubernetes.io/serviceaccount/token", +-- "kubernetes_host": "kubernetes.default.svc", +-- "kubernetes_port": "443", +-- "ssl_verify": true +-- } +-- +-- All fields have sensible defaults for in-cluster usage, so the minimal +-- valid configuration is an empty object `{}`. +-- +-- TLS note: when ssl_verify is true (the default), the TLS handshake is +-- validated against the nginx-level lua_ssl_trusted_certificate bundle. +-- For in-cluster usage, set apisix.ssl.ssl_trusted_certificate in config.yaml +-- to /var/run/secrets/kubernetes.io/serviceaccount/ca.crt so that the +-- kube-apiserver certificate (signed by the cluster CA) is trusted. + +local core = require("apisix.core") +local http = require("resty.http") +local env = core.env + +local io_open = io.open +local find = core.string.find +local sub = core.string.sub +local ngx_decode_base64 = ngx.decode_base64 + +local DEFAULT_SA_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/token" + +local schema = { + type = "object", + properties = { + service_account_file = { + type = "string", + description = "Path to the ServiceAccount token file. " + .. "Defaults to the standard in-cluster path.", + default = DEFAULT_SA_FILE, + }, + kubernetes_host = { + type = "string", + description = "Kubernetes API server hostname or IP. " + .. "Defaults to the KUBERNETES_SERVICE_HOST environment variable.", + }, + kubernetes_port = { + type = "string", + description = "Kubernetes API server port. " + .. "Defaults to the KUBERNETES_SERVICE_PORT environment variable.", + }, + endpoint = { + type = "string", + description = "Full base URL of the Kubernetes API server " + .. "(e.g. https://kubernetes.default.svc:443). " + .. "When set, kubernetes_host and kubernetes_port are ignored. " + .. "Useful for testing with plain HTTP mock servers.", + }, + ssl_verify = { + type = "boolean", + description = "Verify the Kubernetes API server TLS certificate. " + .. "When true, validation uses the nginx-level " + .. "lua_ssl_trusted_certificate bundle; set " + .. "apisix.ssl.ssl_trusted_certificate to the cluster CA " + .. "(/var/run/secrets/kubernetes.io/serviceaccount/ca.crt) " + .. "in config.yaml for in-cluster usage.", + default = true, + }, + }, + required = {}, +} + +local _M = { + schema = schema, +} + + +local function read_file(path) + local f, err = io_open(path, "r") + if not f then + return nil, "failed to open file " .. path .. ": " .. err + end + local content = f:read("*a") + f:close() + if not content or content == "" then + return nil, "file is empty: " .. path + end + return content +end + + +local function make_request_to_k8s(conf, namespace, secret_name) + local sa_file = conf.service_account_file or DEFAULT_SA_FILE + local token, err = read_file(sa_file) + if not token then + return nil, err + end + -- strip trailing newline + token = token:gsub("%s+$", "") + + local k8s_host = conf.kubernetes_host + or env.fetch_by_uri("$ENV://KUBERNETES_SERVICE_HOST") + or os.getenv("KUBERNETES_SERVICE_HOST") + + local k8s_port = conf.kubernetes_port + or env.fetch_by_uri("$ENV://KUBERNETES_SERVICE_PORT") + or os.getenv("KUBERNETES_SERVICE_PORT") + or "443" + + local base_url + if conf.endpoint then + base_url = conf.endpoint + else + if not k8s_host then + return nil, "kubernetes_host is not set and KUBERNETES_SERVICE_HOST env var is missing" + end + base_url = "https://"; .. k8s_host .. ":" .. k8s_port + end + + local uri = base_url + .. "/api/v1/namespaces/" .. namespace + .. "/secrets/" .. secret_name + + core.log.info("fetching Kubernetes secret from: ", uri) + + local httpc = http.new() + httpc:set_timeout(5000) + + local ssl_verify = conf.ssl_verify + if ssl_verify == nil then + ssl_verify = true + end + + local request_opts = { + method = "GET", + headers = { + ["Authorization"] = "Bearer " .. token, + ["Accept"] = "application/json", + }, + ssl_verify = ssl_verify, + } + + local res, req_err = httpc:request_uri(uri, request_opts) + if not res then + return nil, "failed to request Kubernetes API: " .. req_err + end + + if res.status == 401 or res.status == 403 then + return nil, "unauthorized to read Kubernetes secret " + .. namespace .. "/" .. secret_name + .. " (HTTP " .. res.status .. "): check RBAC permissions for the ServiceAccount" + end + + if res.status == 404 then + return nil, "Kubernetes secret not found: " .. namespace .. "/" .. secret_name + end + + if res.status ~= 200 then + return nil, "unexpected HTTP status " .. res.status + .. " from Kubernetes API for secret " + .. namespace .. "/" .. secret_name + end + + return res.body +end + + +-- key format: {namespace}/{secret-name}/{data-key} +local function get(conf, key) + core.log.info("fetching data from Kubernetes secret for key: ", key) + + local idx1 = find(key, "/") + if not idx1 then + return nil, "invalid key format, expected {namespace}/{secret-name}/{data-key}, got: " + .. key + end + + local namespace = sub(key, 1, idx1 - 1) + if namespace == "" then + return nil, "namespace is empty in key: " .. key Review Comment: `namespace` and `secret_name` go into the request path unescaped. `resty.http`'s `_format_request` inserts the path verbatim, and its `parse_uri` splits on `([^\?]*)`, so `?`, `#` and CRLF all pass straight through to the kube-apiserver. A ref like `$secret://kubernetes/mgr/default?labelSelector=x/mysecret/key` emits `GET /api/v1/namespaces/default?labelSelector=x/secrets/mysecret`. This isn't a vulnerability — per the threat model (`docs/en/latest/security-threat-model.md` §4.2/§4.3) the Admin API operator is trusted and already has arbitrary Lua via `script`. But validating against the DNS-1123 rules (namespace `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`, secret name as DNS subdomain) is cheap, kills the class, and turns a malformed ref into a clear error instead of a silently reshaped apiserver call. ########## t/secret/kubernetes.t: ########## @@ -0,0 +1,463 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BEGIN { + $ENV{KUBERNETES_SERVICE_HOST} = "127.0.0.1"; + $ENV{KUBERNETES_SERVICE_PORT} = "6443"; +} + +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); +log_level("info"); + +run_tests; + +__DATA__ + +=== TEST 1: schema validation - all fields valid +--- config + location /t { + content_by_lua_block { + local kubernetes = require("apisix.secret.kubernetes") + local core = require("apisix.core") + + local test_cases = { + {}, + {ssl_verify = true}, + {ssl_verify = false}, + {ssl_verify = 1234}, + {service_account_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"}, + {service_account_file = 1234}, + {kubernetes_host = "kubernetes.default.svc"}, + {kubernetes_host = 1234}, + {kubernetes_port = "443"}, + {kubernetes_port = 1234}, + { + service_account_file = "/var/run/secrets/kubernetes.io/serviceaccount/token", + kubernetes_host = "kubernetes.default.svc", + kubernetes_port = "443", + ssl_verify = true, + }, + } + + for _, conf in ipairs(test_cases) do + local ok, err = core.schema.check(kubernetes.schema, conf) + ngx.say(ok and "done" or err) + end + } + } +--- request +GET /t +--- response_body +done +done +done +property "ssl_verify" validation failed: wrong type: expected boolean, got number +done +property "service_account_file" validation failed: wrong type: expected string, got number +done +property "kubernetes_host" validation failed: wrong type: expected string, got number +done +property "kubernetes_port" validation failed: wrong type: expected string, got number +done + + + +=== TEST 2: get - invalid key format (no slashes) +--- config + location /t { + content_by_lua_block { + local kubernetes = require("apisix.secret.kubernetes") + local data, err = kubernetes.get({}, "no-slashes") + if err then + return ngx.say(err) + end + ngx.say(data) + } + } +--- request +GET /t +--- response_body +invalid key format, expected {namespace}/{secret-name}/{data-key}, got: no-slashes + + + +=== TEST 3: get - invalid key format (only one slash, missing data-key) +--- config + location /t { + content_by_lua_block { + local kubernetes = require("apisix.secret.kubernetes") + local data, err = kubernetes.get({}, "default/my-secret") + if err then + return ngx.say(err) + end + ngx.say(data) + } + } +--- request +GET /t +--- response_body +invalid key format, missing data-key, expected {namespace}/{secret-name}/{data-key}, got: default/my-secret + + + +=== TEST 4: get - empty namespace +--- config + location /t { + content_by_lua_block { + local kubernetes = require("apisix.secret.kubernetes") + local data, err = kubernetes.get({}, "/my-secret/my-key") + if err then + return ngx.say(err) + end + ngx.say(data) + } + } +--- request +GET /t +--- response_body +namespace is empty in key: /my-secret/my-key + + + +=== TEST 5: get - empty secret-name +--- config + location /t { + content_by_lua_block { + local kubernetes = require("apisix.secret.kubernetes") + local data, err = kubernetes.get({}, "default//my-key") + if err then + return ngx.say(err) + end + ngx.say(data) + } + } +--- request +GET /t +--- response_body +secret-name is empty in key: default//my-key + + + +=== TEST 6: get - empty data-key +--- config + location /t { + content_by_lua_block { + local kubernetes = require("apisix.secret.kubernetes") + local data, err = kubernetes.get({}, "default/my-secret/") + if err then + return ngx.say(err) + end + ngx.say(data) + } + } +--- request +GET /t +--- response_body +data-key is empty in key: default/my-secret/ + + + +=== TEST 7: get - service account file not found +--- config + location /t { + content_by_lua_block { + local kubernetes = require("apisix.secret.kubernetes") + local conf = { + service_account_file = "/nonexistent/token", + ssl_verify = false, + } + local data, err = kubernetes.get(conf, "default/my-secret/my-key") + if err then + return ngx.say(err) + end + ngx.say(data) + } + } +--- request +GET /t +--- response_body_like +failed to open file /nonexistent/token:.* + + + +=== TEST 8: get - connection refused (API server not reachable) +--- config + location /t { + content_by_lua_block { + -- Write a temp token file for the test + local f = io.open("/tmp/test-sa-token", "w") + f:write("test-token") + f:close() + + local kubernetes = require("apisix.secret.kubernetes") + local conf = { + service_account_file = "/tmp/test-sa-token", + endpoint = "https://127.0.0.2:9999";, + ssl_verify = false, + } + local data, err = kubernetes.get(conf, "default/my-secret/my-key") + if err then + return ngx.say(err) + end + ngx.say(data) + } + } +--- request +GET /t +--- response_body_like +failed to request Kubernetes API:.* +--- timeout: 6 + + + +=== TEST 9: get - mock Kubernetes API returns 401 +--- config + location /mock-k8s-401 { Review Comment: This test fails in CI, and as written it can't test what its name says. The mock is at `/mock-k8s-401`, but the code requests `/api/v1/namespaces/default/secrets/my-secret` — no matching location, so it hits APISIX's catch-all and gets a 404. `get()` maps that to `Kubernetes secret not found: default/my-secret`, which isn't in the `(unauthorized...|failed to request...|unexpected HTTP status)` alternation. CI: ``` Failed test 'TEST 9: get - mock Kubernetes API returns 401 - response_body_like - response is expected (Kubernetes secret not found: default/my-secret)' ``` Move the mock to `location ~ "^/api/v1/namespaces/default/secrets/my-secret$"` returning 401, and assert the specific message `unauthorized to read Kubernetes secret default/my-secret (HTTP 401)`. The three-way alternation would pass on almost any failure anyway. While you're here: the `BEGIN { $ENV{KUBERNETES_SERVICE_HOST} = ... }` block at the top is dead — no test exercises the env fallback. And a few branches are untested: non-200/401/403/404 status, malformed JSON, a secret with no `data` field, and the `ngx.decode_base64` nil path. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
