This is an automated email from the ASF dual-hosted git repository. lingsamuel pushed a commit to branch cert-manager in repository https://gitbox.apache.org/repos/asf/apisix-ingress-controller.git
commit a28629290a3d57ff828b9effd9513ffed34e0f12 Author: Ling Samuel <[email protected]> AuthorDate: Thu Sep 16 08:24:37 2021 +0800 feat: support cert-manager --- docs/en/latest/practices/cert-manager/ca.yaml | 8 + docs/en/latest/practices/cert-manager/issuer.yaml | 7 + .../manage-certificates-with-cert-manager.md | 237 +++++++++++++++++++++ ...anage-ingress-certificates-with-cert-manager.md | 191 +++++++++++++++++ pkg/ingress/controller.go | 2 + pkg/ingress/secret.go | 61 +++--- pkg/kube/translation/apisix_ssl.go | 63 +++++- pkg/kube/translation/translator.go | 3 + test/e2e/ingress/ingress.go | 54 +++++ test/e2e/ingress/secret.go | 233 ++++++++++++++++++++ test/e2e/scaffold/ssl.go | 20 ++ 11 files changed, 840 insertions(+), 39 deletions(-) diff --git a/docs/en/latest/practices/cert-manager/ca.yaml b/docs/en/latest/practices/cert-manager/ca.yaml new file mode 100644 index 0000000..d751fc7 --- /dev/null +++ b/docs/en/latest/practices/cert-manager/ca.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +type: kubernetes.io/tls +metadata: + name: ca-key-pair +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUY5ekNDQTkrZ0F3SUJBZ0lVRkt1ekFKWmdtL2ZzRlM2SkRyZCtsY3BWWnI4d0RRWUpLb1pJaHZjTkFRRUwKQlFBd2dad3hDekFKQmdOVkJBWVRBa05PTVJFd0R3WURWUVFJREFoYWFHVnFhV0Z1WnpFUk1BOEdBMVVFQnd3SQpTR0Z1WjNwb2IzVXhHREFXQmdOVkJBb01EMEZRU1ZOSldDMVVaWE4wTFVOQlh6RVlNQllHQTFVRUN3d1BRVkJKClUwbFlYME5CWDFKUFQxUmZNUlV3RXdZRFZRUUREQXhCVUVsVFNWZ3VVazlQVkY4eEhEQWFCZ2txaGtpRzl3MEIKQ1FFV0RYUmxjM1JBZEdWemRDNWpiMjB3SGhjTk1qRXdOVEkzTVRNek5qSTRXaGNOTWpJd05USTNNVE16TmpJNApXakNCbkRFT [...] + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUpRd0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQ1Mwd2dna3BBZ0VBQW9JQ0FRQ3lVZEpVRnZ5QWFrRXgKUHptdEQ0dUM1WnBXRkVocDZyUlRhb21jamhkRDQxY3pYcXFBL1NRejRqTlVFY2d2S1RRbmVTLzBJYlZDRERNZApFWlF5UFE1Q2R3Q3lGbmF6eUdaY0tiM2d5WCtDclZ1M00vbm11T2dPa0VNQWVzWE1sY1ZwNmhBWklubXU2cXhCClBEV3BHVGpJNEpLcERkdi9aSllPbDhnT1JWbjc3TThiYmlCN1kzTXRpcXZHekF0STNaT3dxaEdYdUhMZXJOc0oKWENQTGZMUGFhN3pBNXZJSEYrdXNlQUlmZ3R2MXZndDVVNlRBNEJxa2FrajhnbHU1MmxvSjlMVnBrMG45WXNtOApHRE1uWVF2c [...] diff --git a/docs/en/latest/practices/cert-manager/issuer.yaml b/docs/en/latest/practices/cert-manager/issuer.yaml new file mode 100644 index 0000000..f83daa8 --- /dev/null +++ b/docs/en/latest/practices/cert-manager/issuer.yaml @@ -0,0 +1,7 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: Issuer +metadata: + name: ca-issuer +spec: + ca: + secretName: ca-key-pair diff --git a/docs/en/latest/practices/manage-certificates-with-cert-manager.md b/docs/en/latest/practices/manage-certificates-with-cert-manager.md new file mode 100644 index 0000000..37a0322 --- /dev/null +++ b/docs/en/latest/practices/manage-certificates-with-cert-manager.md @@ -0,0 +1,237 @@ +--- +title: Manage Certificates With Cert Manager +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +This tutorial will detail how to manage secrets of ApisixTls using cert-manager. + +## Prerequisites + +* Prepare an available Kubernetes cluster in your workstation, we recommend you to use [Minikube](https://github.com/kubernetes/minikube). +* Install Apache APISIX in Kubernetes by [Helm Chart](https://github.com/apache/apisix-helm-chart). +* Install [apisix-ingress-controller](https://github.com/apache/apisix-ingress-controller/blob/master/install.md). +* Install [cert-manager](https://cert-manager.io/docs/installation/#default-static-install). + + +In this guide, we assume that your APISIX is installed with `ssl` enabled, which is not enabled by default in the Helm Chart. To enable it, you need to set `gateway.tls.enabled=true` during installation. + +Assume that the SSL port is `9443`. + +## Create Issuer + +For testing purposes, we will use a simple CA issuer. All required files can be found [here](./cert-manager). + +To create a CA issuer, use the following commands: + +```bash +kubectl apply -f ./cert-manager/ca.yaml +kubectl apply -f ./cert-manager/issuer.yaml +``` + +If the cert-manager is working correctly, we should be able to see the Ready status by running: + +```bash +kubectl get issuer +``` + +It should output: + +``` +NAME READY AGE +ca-issuer True 50s +``` + +## Create Certificate + +Before creating ApisixTls, we should create a `Certificate` resource. + +```yaml +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: demo-cert +spec: + dnsNames: + - local.httpbin.org + issuerRef: + kind: Issuer + name: ca-issuer + secretName: example-cert + usages: + - digital signature + - key encipherment + renewBefore: 0h55m0s + duration: 1h0m0s +``` + +Note that we set the parameters `duration` and `renewBefore`. We want to test if the certificate rotation functionality is working well, so a shorter renewal time will help. + +Like `Issuer`, we could see its readiness status by running: + +```bash +kubectl get certificate +``` + +It should output: + +``` +NAME READY SECRET AGE +demo-cert True example-cert 50s +``` + +Check the secrets by running: + +```bash +kubectl get secret +``` + +It should output: + +``` +NAME TYPE DATA AGE +example-cert kubernetes.io/tls 3 2m20s +``` + +This means that our cert-manager is working properly. + +## Create Test Service + +We use [kennethreitz/httpbin](https://hub.docker.com/r/kennethreitz/httpbin/) as the service image. + +Deploy it by running: + +```bash +kubectl run httpbin --image kennethreitz/httpbin --port 80 +kubectl expose pod httpbin --port 80 +``` + +## Route the Service + +Create an ApisixRoute to route the service: + +```yaml +apiVersion: apisix.apache.org/v2beta1 +kind: ApisixRoute +metadata: + name: httpserver-route +spec: + http: + - name: httpbin + match: + hosts: + - local.httpbin.org + paths: + - "/*" + backend: + serviceName: httpbin + servicePort: 80 +``` + +Run curl command in a APISIX pod to see if the routing configuration works. + +```bash +kubectl -n <APISIX_NAMESPACE> exec -it <APISIX_POD_NAME> -- curl http://127.0.0.1:9080/ip -H 'Host: local.httpbin.org' +``` + +It should output: + +```json +{ + "origin": "127.0.0.1" +} +``` + +## Secure the Route + +Create an ApisixTls to secure the route, referring to the secret created by cert-manager: + +```yaml +apiVersion: apisix.apache.org/v1 +kind: ApisixTls +metadata: + name: example-tls +spec: + hosts: + - local.httpbin.org + secret: + name: example-cert # the secret created by cert-manager + namespace: default # secret namespace +``` + +Run curl command in a APISIX pod to see if the Ingress and TLS configuration are working. + +```bash +kubectl -n <APISIX_NAMESPACE> exec -it <APISIX_POD_NAME> -- curl --resolve 'local.httpbin.org:9443:127.0.0.1' "https://local.httpbin.org:9443/ip"; -k +``` + +It should output: + +```json +{ + "origin": "127.0.0.1" +} +``` + +## Test Certificate Rotation + +To verify certificate rotation, we can add a verbose parameter `-v` to curl command: + +```bash +kubectl -n <APISIX_NAMESPACE> exec -it <APISIX_POD_NAME> -- curl --resolve 'local.httpbin.org:9443:127.0.0.1' "https://local.httpbin.org:9443/ip"; -k -v +``` + +The verbose option will show us the handshake log, which also contains the certificate information. + +Example output: + +``` +* Added local.httpbin.org:9443:127.0.0.1 to DNS cache +* Hostname local.httpbin.org was found in DNS cache +* Trying 127.0.0.1:9443... +* Connected to local.httpbin.org (127.0.0.1) port 9443 (#0) +... +... +* Server certificate: +* subject: [NONE] +* start date: Sep 16 00:14:55 2021 GMT +* expire date: Sep 16 01:14:55 2021 GMT +* issuer: C=CN; ST=Zhejiang; L=Hangzhou; O=APISIX-Test-CA_; OU=APISIX_CA_ROOT_; CN=APISIX.ROOT_; [email protected] +``` + +We could see the start date and expiration date of the server certificate. + +Since the `Certificate` we defined requires the cert-manager to renew the cert every 5 minutes, we should be able to see the changes to the server certificate after 5 minutes. + +``` +* Added local.httpbin.org:9443:127.0.0.1 to DNS cache +* Hostname local.httpbin.org was found in DNS cache +* Trying 127.0.0.1:9443... +* Connected to local.httpbin.org (127.0.0.1) port 9443 (#0) +... +... +* Server certificate: +* subject: [NONE] +* start date: Sep 16 00:19:55 2021 GMT +* expire date: Sep 16 01:19:55 2021 GMT +* issuer: C=CN; ST=Zhejiang; L=Hangzhou; O=APISIX-Test-CA_; OU=APISIX_CA_ROOT_; CN=APISIX.ROOT_; [email protected] +``` + +The certificate was rotated as expected. diff --git a/docs/en/latest/practices/manage-ingress-certificates-with-cert-manager.md b/docs/en/latest/practices/manage-ingress-certificates-with-cert-manager.md new file mode 100644 index 0000000..20ed983 --- /dev/null +++ b/docs/en/latest/practices/manage-ingress-certificates-with-cert-manager.md @@ -0,0 +1,191 @@ +--- +title: Manage Ingress Certificates With Cert Manager +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +This tutorial will detail how to secure ingress using cert-manager. + +## Prerequisites + +* Prepare an available Kubernetes cluster in your workstation, we recommend you to use [Minikube](https://github.com/kubernetes/minikube). +* Install Apache APISIX in Kubernetes by [Helm Chart](https://github.com/apache/apisix-helm-chart). +* Install [apisix-ingress-controller](https://github.com/apache/apisix-ingress-controller/blob/master/install.md). +* Install [cert-manager](https://cert-manager.io/docs/installation/#default-static-install). + + +In this guide, we assume that your APISIX is installed with `ssl` enabled, which is not enabled by default in the Helm Chart. To enable it, you need to set `gateway.tls.enabled=true` during installation. + +Assume that the SSL port is `9443`. + +## Create Issuer + +For testing purposes, we will use a simple CA issuer. All required files can be found [here](./cert-manager). + +To create a CA issuer, use the following commands: + +```bash +kubectl apply -f ./cert-manager/ca.yaml +kubectl apply -f ./cert-manager/issuer.yaml +``` + +If the cert-manager is working correctly, we should be able to see the Ready status by running: + +```bash +kubectl get issuer +``` + +It should output: + +``` +NAME READY AGE +ca-issuer True 50s +``` + +## Create Test Certificate + +To ensure that cert-manager is working properly, we can create a test `Certificate` resource. + +```yaml +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: demo-cert +spec: + dnsNames: + - example.com + issuerRef: + kind: Issuer + name: ca-issuer + secretName: example-cert + usages: + - digital signature + - key encipherment +``` + +Like `Issuer`, we could see its readiness status by running: + +```bash +kubectl get certificate +``` + +It should output: + +``` +NAME READY SECRET AGE +demo-cert True example.com 50s +``` + +Check the secrets by running: + +```bash +kubectl get secret +``` + +It should output: + +``` +NAME TYPE DATA AGE +example.com kubernetes.io/tls 3 2m20s +``` + +This means that our cert-manager is working properly. + +## Create Test Service + +We use [kennethreitz/httpbin](https://hub.docker.com/r/kennethreitz/httpbin/) as the service image. + +Deploy it by running: + +```bash +kubectl run httpbin --image kennethreitz/httpbin --port 80 +kubectl expose pod httpbin --port 80 +``` + +## Secure Ingress + +The cert-manager supports several ways to [secure ingress](https://cert-manager.io/docs/usage/ingress/). The easiest way is to use annotations. + +By using annotations, we don't need to manage `Certificate` CRD manually. + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: httpserver-ingress + annotations: + # add an annotation indicating the issuer to use. + cert-manager.io/issuer: "ca-issuer" +spec: + # apisix-ingress-controller is only interested in Ingress + # resources with the matched ingressClass name, in our case, + # it's apisix. + ingressClassName: apisix + tls: + - hosts: + - local.httpbin.org # placing a host in the TLS config will determine what ends up in the cert's subjectAltNames + secretName: ingress-cert-manager-tls # cert-manager will store the created certificate in this secret. + rules: + - host: local.httpbin.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: httpbin + port: + number: 80 +``` + +The annotation `cert-manager.io/issuer` tells cert-manager which issuer should be used. The Issuer must be in the same namespace as the Ingress resource. Please read [Securing Ingress Resources](https://cert-manager.io/docs/usage/ingress/) for more details. + +We should now be able to see the certificate and secret resource created by cert-manager: + +```bash +kubectl get certificate +kubectl get secret +``` + +It should output: + +``` +NAME READY SECRET AGE +ingress-cert-manager-tls True ingress-cert-manager-tls 2m + +NAME TYPE DATA AGE +ingress-cert-manager-tls kubernetes.io/tls 3 3m +``` + +## Test + +Run curl command in a APISIX pod to see if the Ingress and TLS configuration works. + +```bash +kubectl -n <APISIX_NAMESPACE> exec -it <APISIX_POD_NAME> -- curl --resolve 'local.httpbin.org:9443:127.0.0.1' "https://local.httpbin.org:9443/ip"; -k +``` + +It should output: + +```json +{ + "origin": "127.0.0.1" +} +``` diff --git a/pkg/ingress/controller.go b/pkg/ingress/controller.go index 2af856d..0b83a8e 100644 --- a/pkg/ingress/controller.go +++ b/pkg/ingress/controller.go @@ -81,6 +81,8 @@ type Controller struct { recorder record.EventRecorder // this map enrolls which ApisixTls objects refer to a Kubernetes // Secret object. + // type: Map<SecretKey, Map<ApisixTlsKey, ApisixTls>> + // SecretKey is `namespace_name`, ApisixTlsKey is kube style meta key: `namespace/name` secretSSLMap *sync.Map // leaderContextCancelFunc will be called when apisix-ingress-controller diff --git a/pkg/ingress/secret.go b/pkg/ingress/secret.go index 700bdd7..e0d0333 100644 --- a/pkg/ingress/secret.go +++ b/pkg/ingress/secret.go @@ -18,6 +18,7 @@ package ingress import ( "context" "fmt" + v1 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v1" "sync" "time" @@ -28,7 +29,6 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" - "github.com/apache/apisix-ingress-controller/pkg/kube/translation" "github.com/apache/apisix-ingress-controller/pkg/log" "github.com/apache/apisix-ingress-controller/pkg/types" apisixv1 "github.com/apache/apisix-ingress-controller/pkg/types/apisix/v1" @@ -95,12 +95,10 @@ func (c *secretController) sync(ctx context.Context, ev *types.Event) error { return err } sec, err := c.controller.secretLister.Secrets(namespace).Get(name) - - secretMapkey := namespace + "_" + name if err != nil { if !k8serrors.IsNotFound(err) { log.Errorw("failed to get Secret", - zap.String("key", secretMapkey), + zap.String("key", key), zap.Error(err), ) return err @@ -108,7 +106,7 @@ func (c *secretController) sync(ctx context.Context, ev *types.Event) error { if ev.Type != types.EventDelete { log.Warnw("Secret was deleted before it can be delivered", - zap.String("key", secretMapkey), + zap.String("key", key), ) return nil } @@ -119,15 +117,16 @@ func (c *secretController) sync(ctx context.Context, ev *types.Event) error { // that means object with same namespace and name was created, discarding // this stale DELETE event. log.Warnw("discard the stale secret delete event since the resource still exists", - zap.String("key", secretMapkey), + zap.String("key", key), ) return nil } sec = ev.Tombstone.(*corev1.Secret) } + + secretMapKey := namespace + "_" + name // sync SSL in APISIX which is store in secretSSLMap - // FixMe Need to update the status of CRD ApisixTls - ssls, ok := c.controller.secretSSLMap.Load(secretMapkey) + ssls, ok := c.controller.secretSSLMap.Load(secretMapKey) if !ok { // This secret is not concerned. return nil @@ -143,26 +142,25 @@ func (c *secretController) sync(ctx context.Context, ev *types.Event) error { } tls, err := c.controller.apisixTlsLister.ApisixTlses(tlsNamespace).Get(tlsName) if err != nil { - log.Warnw("secret related ApisixTls resource not found, skip", + log.Debugw("secret related ApisixTls resource not found, skip", zap.String("ApisixTls", tlsMetaKey), ) return true } + + // We don't expect a secret to be used as both SSL and mTLS in ApisixTls if tls.Spec.Secret.Namespace == sec.Namespace && tls.Spec.Secret.Name == sec.Name { - cert, ok := sec.Data["cert"] - if !ok { - log.Warnw("secret required by ApisixTls invalid", - zap.String("ApisixTls", tlsMetaKey), - zap.Error(translation.ErrEmptyCert), - ) - return true - } - pkey, ok := sec.Data["key"] - if !ok { - log.Warnw("secret required by ApisixTls invalid", + cert, pkey, err := c.controller.translator.ExtractKeyPair(sec, true) + if err != nil { + log.Errorw("secret required by ApisixTls invalid", zap.String("ApisixTls", tlsMetaKey), - zap.Error(translation.ErrEmptyPrivKey), + zap.Error(err), ) + go func(tls *v1.ApisixTls) { + c.controller.recorderEventS(tls, corev1.EventTypeWarning, _resourceSyncAborted, + fmt.Sprintf("sync from secret %s changes failed, error: %s", key, err.Error())) + c.controller.recordStatus(tls, _resourceSyncAborted, err, metav1.ConditionFalse) + }(tls) return true } // sync ssl @@ -170,19 +168,24 @@ func (c *secretController) sync(ctx context.Context, ev *types.Event) error { ssl.Key = string(pkey) } else if tls.Spec.Client != nil && tls.Spec.Client.CASecret.Namespace == sec.Namespace && tls.Spec.Client.CASecret.Name == sec.Name { - ca, ok := sec.Data["cert"] - if !ok { - log.Warnw("secret required by ApisixTls invalid", - zap.String("resource", tlsMetaKey), - zap.Error(translation.ErrEmptyCert), + ca, _, err := c.controller.translator.ExtractKeyPair(sec, false) + if err != nil { + log.Errorw("ca secret required by ApisixTls invalid", + zap.String("ApisixTls", tlsMetaKey), + zap.Error(err), ) + go func(tls *v1.ApisixTls) { + c.controller.recorderEventS(tls, corev1.EventTypeWarning, _resourceSyncAborted, + fmt.Sprintf("sync from ca secret %s changes failed, error: %s", key, err.Error())) + c.controller.recordStatus(tls, _resourceSyncAborted, err, metav1.ConditionFalse) + }(tls) return true } ssl.Client = &apisixv1.MutualTLSClientConfig{ CA: string(ca), } } else { - log.Warnw("stale secret cache, ApisixTls doesn't requires target secret", + log.Infow("stale secret cache, ApisixTls doesn't requires target secret", zap.String("ApisixTls", tlsMetaKey), zap.String("secret", key), ) @@ -190,7 +193,7 @@ func (c *secretController) sync(ctx context.Context, ev *types.Event) error { } // Use another goroutine to send requests, to avoid // long time lock occupying. - go func(ssl *apisixv1.Ssl) { + go func(ssl *apisixv1.Ssl, tls *v1.ApisixTls) { err := c.controller.syncSSL(ctx, ssl, ev.Type) if err != nil { log.Errorw("failed to sync ssl to APISIX", @@ -206,7 +209,7 @@ func (c *secretController) sync(ctx context.Context, ev *types.Event) error { fmt.Sprintf("sync from secret %s changes", key)) c.controller.recordStatus(tls, _resourceSynced, nil, metav1.ConditionTrue) } - }(ssl) + }(ssl, tls) return true }) return err diff --git a/pkg/kube/translation/apisix_ssl.go b/pkg/kube/translation/apisix_ssl.go index 7ab54fa..bf1bfe9 100644 --- a/pkg/kube/translation/apisix_ssl.go +++ b/pkg/kube/translation/apisix_ssl.go @@ -17,12 +17,16 @@ package translation import ( "errors" + v1 "k8s.io/api/core/v1" + "github.com/apache/apisix-ingress-controller/pkg/id" configv1 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v1" apisixv1 "github.com/apache/apisix-ingress-controller/pkg/types/apisix/v1" ) var ( + // ErrUnknownSecretFormat means the secret doesn't contain required fields + ErrUnknownSecretFormat = errors.New("unknown secret format") // ErrEmptyCert means the cert field in Kubernetes Secret is not found. ErrEmptyCert = errors.New("missing cert field") // ErrEmptyPrivKey means the key field in Kubernetes Secret is not found. @@ -34,14 +38,11 @@ func (t *translator) TranslateSSL(tls *configv1.ApisixTls) (*apisixv1.Ssl, error if err != nil { return nil, err } - cert, ok := s.Data["cert"] - if !ok { - return nil, ErrEmptyCert - } - key, ok := s.Data["key"] - if !ok { - return nil, ErrEmptyPrivKey + cert, key, err := t.ExtractKeyPair(s, true) + if err != nil { + return nil, err } + var snis []string for _, host := range tls.Spec.Hosts { snis = append(snis, string(host)) @@ -61,9 +62,9 @@ func (t *translator) TranslateSSL(tls *configv1.ApisixTls) (*apisixv1.Ssl, error if err != nil { return nil, err } - ca, ok := caSecret.Data["cert"] - if !ok { - return nil, ErrEmptyCert + ca, _, err := t.ExtractKeyPair(caSecret, false) + if err != nil { + return nil, err } ssl.Client = &apisixv1.MutualTLSClientConfig{ CA: string(ca), @@ -73,3 +74,45 @@ func (t *translator) TranslateSSL(tls *configv1.ApisixTls) (*apisixv1.Ssl, error return ssl, nil } + +func (t *translator) ExtractKeyPair(s *v1.Secret, hasPrivateKey bool) ([]byte, []byte, error) { + if _, ok := s.Data["cert"]; ok { + return t.extractApisixSecretKeyPair(s, hasPrivateKey) + } else if _, ok := s.Data[v1.TLSCertKey]; ok { + return t.extractKubeSecretKeyPair(s, hasPrivateKey) + } else { + return nil, nil, ErrUnknownSecretFormat + } +} + +func (t *translator) extractApisixSecretKeyPair(s *v1.Secret, hasPrivateKey bool) (cert []byte, key []byte, err error) { + var ok bool + cert, ok = s.Data["cert"] + if !ok { + return nil, nil, ErrEmptyCert + } + + if hasPrivateKey { + key, ok = s.Data["key"] + if !ok { + return nil, nil, ErrEmptyPrivKey + } + } + return +} + +func (t *translator) extractKubeSecretKeyPair(s *v1.Secret, hasPrivateKey bool) (cert []byte, key []byte, err error) { + var ok bool + cert, ok = s.Data[v1.TLSCertKey] + if !ok { + return nil, nil, ErrEmptyCert + } + + if hasPrivateKey { + key, ok = s.Data[v1.TLSPrivateKeyKey] + if !ok { + return nil, nil, ErrEmptyPrivKey + } + } + return +} diff --git a/pkg/kube/translation/translator.go b/pkg/kube/translation/translator.go index c1f6ac9..4d0857d 100644 --- a/pkg/kube/translation/translator.go +++ b/pkg/kube/translation/translator.go @@ -88,6 +88,9 @@ type Translator interface { // TranslateApisixConsumer translates the configv2alpha1.APisixConsumer object into the APISIX Consumer // resource. TranslateApisixConsumer(*configv2alpha1.ApisixConsumer) (*apisixv1.Consumer, error) + // ExtractKeyPair extracts certificate and private key pair from secret + // Supports APISIX style ("cert" and "key") and Kube style ("tls.crt" and "tls.key) + ExtractKeyPair(s *corev1.Secret, hasPrivateKey bool) ([]byte, []byte, error) } // TranslatorOptions contains options to help Translator diff --git a/test/e2e/ingress/ingress.go b/test/e2e/ingress/ingress.go index 7d3d45b..8905730 100644 --- a/test/e2e/ingress/ingress.go +++ b/test/e2e/ingress/ingress.go @@ -261,6 +261,60 @@ spec: s.NewAPISIXHttpsClientWithCertificates(host, true, caCertPool, []tls.Certificate{}). GET("/ip").WithHeader("Host", host).Expect().Status(http.StatusOK) }) + + ginkgo.It("should support ingress v1 with kube style tls secret", func() { + // create secrets + err := s.NewKubeTlsSecret(serverCertSecret, serverCert, serverKey) + assert.Nil(ginkgo.GinkgoT(), err, "create server cert secret error") + + // create ingress + host := "mtls.httpbin.local" + // create route + backendSvc, backendSvcPort := s.DefaultHTTPBackend() + ing := fmt.Sprintf(` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: httpbin-ingress-https + annotations: + kubernetes.io/ingress.class: apisix +spec: + tls: + - hosts: + - %s + secretName: %s + rules: + - host: %s + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: %s + port: + number: %d +`, host, serverCertSecret, host, backendSvc, backendSvcPort[0]) + assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(ing)) + time.Sleep(10 * time.Second) + + apisixRoutes, err := s.ListApisixRoutes() + assert.Nil(ginkgo.GinkgoT(), err, "list routes error") + assert.Len(ginkgo.GinkgoT(), apisixRoutes, 1, "route number not expect") + + apisixSsls, err := s.ListApisixSsl() + assert.Nil(ginkgo.GinkgoT(), err, "list SSLs error") + assert.Len(ginkgo.GinkgoT(), apisixSsls, 1, "SSL number should be 1") + assert.Equal(ginkgo.GinkgoT(), id.GenID(s.Namespace()+"_httpbin-ingress-https-tls"), apisixSsls[0].ID, "SSL name") + assert.Equal(ginkgo.GinkgoT(), apisixSsls[0].Snis, []string{host}, "SSL configuration") + + caCertPool := x509.NewCertPool() + ok := caCertPool.AppendCertsFromPEM([]byte(rootCA)) + assert.True(ginkgo.GinkgoT(), ok, "Append cert to CA pool") + + s.NewAPISIXHttpsClientWithCertificates(host, true, caCertPool, []tls.Certificate{}). + GET("/ip").WithHeader("Host", host).Expect().Status(http.StatusOK) + }) }) var _ = ginkgo.Describe("support ingress.networking/v1", func() { diff --git a/test/e2e/ingress/secret.go b/test/e2e/ingress/secret.go index 421a6a2..8717107 100644 --- a/test/e2e/ingress/secret.go +++ b/test/e2e/ingress/secret.go @@ -269,4 +269,237 @@ UnBVSIGJ/c0AhVSDuOAJiF36pvsDysTZXMTFE/9i5bkGOiwtzRNe4Hym/SEZUCpn assert.Nil(ginkgo.GinkgoT(), err, "list tls error") assert.Len(ginkgo.GinkgoT(), tls, 0, "tls number not expect") }) + + ginkgo.It("create a kube style SSL and then update secret ", func() { + backendSvc, backendSvcPort := s.DefaultHTTPBackend() + apisixRoute := fmt.Sprintf(` +apiVersion: apisix.apache.org/v2alpha1 +kind: ApisixRoute +metadata: + name: httpbin-route +spec: + http: + - name: rule1 + match: + hosts: + - api6.com + paths: + - /ip + backend: + serviceName: %s + servicePort: %d +`, backendSvc, backendSvcPort[0]) + assert.Nil(ginkgo.GinkgoT(), s.CreateResourceFromString(apisixRoute)) + + secretName := "test-apisix-tls" + cert := `-----BEGIN CERTIFICATE----- +MIIFcjCCA1qgAwIBAgIJALDqPppBVXQ3MA0GCSqGSIb3DQEBCwUAMGUxCzAJBgNV +BAYTAkNOMRAwDgYDVQQIDAdKaWFuZ3N1MQ8wDQYDVQQHDAZTdXpob3UxEDAOBgNV +BAoMB2FwaTcuYWkxEDAOBgNVBAsMB2FwaTcuYWkxDzANBgNVBAMMBmp3LmNvbTAg +Fw0yMTA0MDkwNzEyMDBaGA8yMDUxMDQwMjA3MTIwMFowZTELMAkGA1UEBhMCQ04x +EDAOBgNVBAgMB0ppYW5nc3UxDzANBgNVBAcMBlN1emhvdTEQMA4GA1UECgwHYXBp +Ny5haTEQMA4GA1UECwwHYXBpNy5haTEPMA0GA1UEAwwGancuY29tMIICIjANBgkq +hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAuEPPnUMlSw41CTdxUNxkQ4gAZ7cPotwY +Y6sVLGtWoR8gKFSZImQIor3UF+8HhN/ZOFRv5tSeeQ/MTE72cs2T5mp6eRU8OrSV +0npk/TpZfaCx7zobsfXB4YU1NZcVWKthFF5X8p//l5gxUMU8V4a01P0aDTmZ67wG +3Fhr0AC067GVYvdwp1yRt6TUUk8ha7JsiySchUIFhX5QMWmrSNhc1bDnHetejMFl +itFFPZkeYG89O/7Ca1K3ca/VVu+/IJ4h7fbF3rt4uP182cJdHl1L94dQSKCJukaW +v+xauWnm4hxOzBK7ImpYB/2eP2D33tmuCLeSv4S+bTG1l7hIN9C/xrYPzfun415h +M2jMK69aAkQL71xa+66kVxioJyNYogYz3ss5ruzDL8K/7bkdO0Zzqldd2+j8lkTl +X4csA+aMHF3v/U7hL/4Wdwi8ziwToRMq9KK9vuh+mPgcdtFGFml+sU+NQfJNm/BN +7fRMZKDIHLacSPE0GUkfW+x3dXOP2lWSZe/iOBZ0NOGNdrOnxTRTr7IH7DYU8aXF +w2GqfAFEQbD4wazCh1AI8lkZr6mPGB1q+HnF2IF7kkgXBHtI5U2KErgcX5BirIVe +v0Yg/OxbbymeTh/hNCcY1kJ1YUCbm9U3U6ZV+d8lj7dQHtugcAzWxSTwpBLVUPrO +eFuhSMLVblUCAwEAAaMjMCEwHwYDVR0RBBgwFoIIYXBpNi5jb22CCiouYXBpNi5j +b20wDQYJKoZIhvcNAQELBQADggIBAFgeSuMPpriYUrQByO3taiY1+56s+KoAmsyA +LH15n2+thAgorusX2g1Zd7UNBHBtVO8c+ihpQb+2PnmgTTGT70ndpRbV5F6124Mt +Hui/X0kjm76RYd1QKN1VFp0Zo+JVdRa+VhDsXWjO0VetINmFhNINFEJctyeHB8oi +aaDL0wZrevHh47hBqtnrmLl+QVG34aLBRhZ5953leiNvXHUJNaT0nLgf0j9p4esS +b2bx9uP4pFI1T9wcv/TE3K0rQbu/uqGY6MgznXHyi4qIK/I+WCa3fF2UZ5P/5EUM +k2ptQneYkLLUVwwmj8C04bYhYe7Z6jkYYp17ojxIP+ejOY1eiE8SYKNjKinmphrM +5aceqIyfbE4TPqvicNzIggA4yEMPbTA0PC/wqbCf61oMc15hwacdeIaQGiwsM+pf +DTk+GBxp3megx/+0XwTQbguleTlHnaaES+ma0vbl6a1rUK0YAUDUrcfFLv6EFtGD +6EHxFf7gH9sTfc2RiGhKxUhRbyEree+6INAvXy+AymVYmQmKuAFqkDQJ+09bTfm8 +bDs+00FijzHFBvC8cIhNffj0qqiv35g+9FTwnE9qpunlrtKG/sMgEXX6m8kL1YQ8 +m5DPGhyEZyt5Js2kzzo8TyINPKmrqriYuiD4p4EH13eSRs3ayanQh6ckC7lb+WXq +3IrSc5hO +-----END CERTIFICATE-----` + key := `-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAuEPPnUMlSw41CTdxUNxkQ4gAZ7cPotwYY6sVLGtWoR8gKFSZ +ImQIor3UF+8HhN/ZOFRv5tSeeQ/MTE72cs2T5mp6eRU8OrSV0npk/TpZfaCx7zob +sfXB4YU1NZcVWKthFF5X8p//l5gxUMU8V4a01P0aDTmZ67wG3Fhr0AC067GVYvdw +p1yRt6TUUk8ha7JsiySchUIFhX5QMWmrSNhc1bDnHetejMFlitFFPZkeYG89O/7C +a1K3ca/VVu+/IJ4h7fbF3rt4uP182cJdHl1L94dQSKCJukaWv+xauWnm4hxOzBK7 +ImpYB/2eP2D33tmuCLeSv4S+bTG1l7hIN9C/xrYPzfun415hM2jMK69aAkQL71xa ++66kVxioJyNYogYz3ss5ruzDL8K/7bkdO0Zzqldd2+j8lkTlX4csA+aMHF3v/U7h +L/4Wdwi8ziwToRMq9KK9vuh+mPgcdtFGFml+sU+NQfJNm/BN7fRMZKDIHLacSPE0 +GUkfW+x3dXOP2lWSZe/iOBZ0NOGNdrOnxTRTr7IH7DYU8aXFw2GqfAFEQbD4wazC +h1AI8lkZr6mPGB1q+HnF2IF7kkgXBHtI5U2KErgcX5BirIVev0Yg/OxbbymeTh/h +NCcY1kJ1YUCbm9U3U6ZV+d8lj7dQHtugcAzWxSTwpBLVUPrOeFuhSMLVblUCAwEA +AQKCAgApTupoMvlVTiYNnuREYGQJz59noN5cgELndR8WCiotjLDE2dJKp2pYMX4u +r2NcImKsAiHj+Z5dPXFrWfhd3EBf01cJdf0+m+VKfi3NpxsQ0smQ+9Hhn1qLmDVJ +gklCy4jD7DKDLeM6tN+5X74bUROQ+/yvIk6jTk+rbhcdVks422LGAPq8SkBQjx8a +JKs1XZZ/ywFbzmU2fA62RR4lAnwtW680QeO8Yk7FRAzltkHdFJMBtCcZsD13uxd0 +meKbCVhJ5JyPRi/WKN2oY65EdF3na+pPnc3CeLiq5e2gy2D7J6VyknBpUrXRdMXZ +J3/p8ZrWUXEQhk26ZP50uNdXy/Bx1jYe+U8mpkTMYVYxgu5K4Zea3yJyRn2piiE/ +9LnKNy/KsINt/0QE55ldvtciyP8RDA/08eQX0gvtKWWC/UFVRZCeL48bpqLmdAfE +cMwlk1b0Lmo2PxULFLMAjaTKmcMAbwl53YRit0MtvaIOwiZBUVHE0blRiKC2DMKi +SA6xLbaYJVDaMcfix8kZkKbC0xBNmL4123qX4RF6IUTPufyUTz/tpjzH6eKDw/88 +LmSx227d7/qWp5gROCDhZOPhtr4rj70JKNqcXUX9iFga+dhloOwfHYjdKndKOLUI +Gp3K9YkPT/fCfesrguUx8BoleO5pC6RQJhRsemkRGlSY8U1ZsQKCAQEA5FepCn1f +A46GsBSQ+/pbaHlbsR2syN3J5RmAFLFozYUrqyHE/cbNUlaYq397Ax7xwQkiN3F2 +z/whTxOh4Sk/HdDF4d+I0PZeoxINxgfzyYkx8Xpzn2loxsRO8fb3g+mOxZidjjXv +vxqUBaj3Y01Ig0UXuw7YqCwys+xg3ELtvcGLBW/7NWMo8zqk2nUjhfcwp4e4AwBt +Xcsc2aShUlr/RUrJH4adjha6Yaqc/8xTXHW8gZi5L2lucwB0LA+CBe4ES9BZLZdX +N575/ohXRdjadHKYceYHiamt2326DzaxVJu2EIXU8dgdgOZ/6krITzuePRQHLPHX +6bDfdg/WSpFrtwKCAQEAzpVqBcJ1fAI7bOkt89j2zZb1r5uD2f9sp/lA/Dj65QKV +ShWR7Y6Jr4ShXmFvIenWtjwsl86PflMgtsJefOmLyv8o7PL154XD8SnNbBlds6IM +MyNKkOJFa5NOrsal7TitaTvtYdKq8Zpqtxe+2kg80wi+tPVQNQS/drOpR3rDiLIE +La/ty8XDYZsSowlzBX+uoFq7GuMct1Uh2T0/I4Kf0ZLXwYjkRlRk4LrU0BRPhRMu +MHugOTYFKXShE2a3OEcxqCgvQ/3pn2TV92pPVKBIBGL6uKUwmXQYKaV3G4u10pJ4 +axq/avBOErcKZOReb0SNiOjiIsth8o+hnpYPU5COUwKCAQBksVNV0NtpUhyK4Ube +FxTgCUQp4pAjM8qoQIp+lY1FtAgBuy6HSneYa59/YQP56FdrbH+uO1bNeL2nhVzJ +UcsHdt0MMeq/WyV4e6mfPjp/EQT5G6qJDY6quD6n7ORRQ1k2QYqY/6fteeb0aAJP +w/DKElnYnz9jSbpCJWbBOrJkD0ki6LK6ZDPWrnGr9CPqG4tVFUBL8pBH4B2kzDhn +fME86TGvuUkZM2SVVQtOsefAyhqKe7KN+cw+4mBYXa5UtxUl6Yap2CcZ2/0aBT2X +C32qBC69a1a/mheUxuiZdODWEqRCvQGedFLuWLbntnqGlh+9h2tyomM4JkskYO96 +io4ZAoIBAFouLW9AOUseKlTb4dx+DRcoXC4BpGhIsWUOUQkJ0rSgEQ2bJu3d+Erv +igYKYJocW0eIMys917Qck75UUS0UQpsmEfaGBUTBRw0C45LZ6+abydmVAVsH+6f/ +USzIuOw6frDeoTy/2zHG5+jva7gcKrkxKxcRs6bBYNdvjGkQtUT5+Qr8rsDyntz/ +9f3IBTcUSuXjVaRiGkoJ1tHfg617u0qgYKEyofv1oWfdB0Oiaig8fEBb51CyPUSg +jiRLBZaCtbGjgSacNB0JxsHP3buikG2hy7NJIVMLs/SSL9GNhpzapciTj5YeOua+ +ksICUxsdgO+QQg9QW3yoqLPy69Pd2dMCggEBANDLoUf3ZE7Dews6cfIa0WC33NCV +FkyECaF2MNOp5Q9y/T35FyeA7UeDsTZ6Dy++XGW4uNStrSI6dCyQARqJw+i7gCst +2m5lIde01ptzDQ9iO1Dv1XasxX/589CyLq6CxLfRgPMJPDeUEg0X7+a0lBT5Hpwk +gNnZmws4l3i7RlVMtACCenmof9VtOcMK/9Qr502WHEoGkQR1r6HZFb25841cehL2 +do+oXlr8db++r87a8QQUkizzc6wXD9JffBNo9AO9Ed4HVOukpEA0gqVGBu85N3xW +jW4KB95bGOTa7r7DM1Up0MbAIwWoeLBGhOIXk7inurZGg+FNjZMA5Lzm6qo= +-----END RSA PRIVATE KEY-----` + // key compare + keyCompare := "HrMHUvE9Esvn7GnZ+vAynaIg/8wlB3r0zm0htmnwofaOw61M98WSdvoWLaQa8YKSdemgQUz2W4MYk2rRZcVSzHfJOLRG7g4ieZau6peDYOmPmp/0ZZFpOzBKoWHN3QP/8i/7SF+JX+EDLD2JO2+GM6iR3f2Zj7v0vx+CcoQ1rjxaXNETSSHo8yvW6pdFZOLgJk4rOHKGypnqzygxxamM8Hq7WSPrWhNe47y1QAfz42kBQXRUJpNNd7W749cTsMWCqBlR+8klTlnSFHkjyijBZjg5ihqZsi/8JzHGrmAixZ54ugPgbufD0/ZJdo3w7opJc4WTnUI2GhiBL+ENCA0X1s/6H8JG8zsC50PvxOBpRgK455TTvejm1JHyt0GTh7c4WFEeQSrbEFzS89BpVrPtre2enO38pkILI8ty8r6tIbZzuOJhM6ZpxQQcAe8OUvFuIIlx21yBvlljbu3eH5Hg7X+wtJR [...] + // create kube secret + err := s.NewKubeTlsSecret(secretName, cert, key) + assert.Nil(ginkgo.GinkgoT(), err, "create secret error") + // create ApisixTls resource + tlsName := "tls-name" + host := "api6.com" + err = s.NewApisixTls(tlsName, host, secretName) + assert.Nil(ginkgo.GinkgoT(), err, "create tls error") + // check ssl in APISIX + time.Sleep(10 * time.Second) + tls, err := s.ListApisixSsl() + assert.Nil(ginkgo.GinkgoT(), err, "list tls error") + assert.Len(ginkgo.GinkgoT(), tls, 1, "tls number not expect") + assert.Equal(ginkgo.GinkgoT(), cert, tls[0].Cert, "tls cert not expect") + assert.Equal(ginkgo.GinkgoT(), keyCompare, tls[0].Key, "tls key not expect") + + // check DP + s.NewAPISIXHttpsClient(host).GET("/ip").WithHeader("Host", host).Expect().Status(http.StatusOK).Body().Raw() + + certUpdate := `-----BEGIN CERTIFICATE----- +MIIFcjCCA1qgAwIBAgIJAM7zkxmhGdNEMA0GCSqGSIb3DQEBCwUAMGUxCzAJBgNV +BAYTAkNOMRAwDgYDVQQIDAdKaWFuZ3N1MQ8wDQYDVQQHDAZTdXpob3UxEDAOBgNV +BAoMB2FwaTcuYWkxEDAOBgNVBAsMB2FwaTcuYWkxDzANBgNVBAMMBmp3LmNvbTAg +Fw0yMTA0MDkwNzE1MzNaGA8yMDUxMDQwMjA3MTUzM1owZTELMAkGA1UEBhMCQ04x +EDAOBgNVBAgMB0ppYW5nc3UxDzANBgNVBAcMBlN1emhvdTEQMA4GA1UECgwHYXBp +Ny5haTEQMA4GA1UECwwHYXBpNy5haTEPMA0GA1UEAwwGancuY29tMIICIjANBgkq +hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAlXGEL4P5+mjllbb4VxkpqjsHny2wY7QZ +NbA0INfLdBq3B5avL0Gp/nHk1iOMq+RkUu+DDb9i0wKw0j1ygaaN4WNXCa7/LZ7E +IVDKcCKF7JQUi5za9A36nTBMLb7ZbFwxnxGkCK2AwdIbUIB6Ciq/d+WEbZelspPQ +PtG9e8pVYTbNhRublGgVD+LEUHYdvSneveFQxdmHhNQMPXNbh+Qmgz48n7cLMEHh +NwRyrWOb3zhdDeKlTm/yfPAQwvQNet+Ol3g19tKhS7WzDndrPfPNYAsoOdXZ5TTY +AlXzwsbVYSOW4gMW39p68ZnoseQHBVZkiTfN2CGvWbzd9gUXOiQship+xRfSoVEY +5FN1KsTIA5fdDbgqCTM4Ed1FTq39hJSBnsQObzQtfjgz0Sivwp9q0NoXX+YM6eBJ +1kzVx6r70UMq+NJbs7mvAW9Ehzv2m2GsinMq3lyvVwSbsJWX9vOHKxmuWympmOw1 +7sUcH4DNHCxnzLK0L0oDV2atpNyKFvmMWSSmw09wOvuE+G+CLwILvgiSiTRPe1Gm +cgaTC92nZsafA5d8r2uxxJRHD7PAQSE5YnsdLsXAFSegYyHss2jq8lzVFwWhTk4t +A/8si8qUHh/hHKt71dYrSRmYTTyj834P8KDwQAMYBGcThtZa7PudOR22toohyte7 +Y7j4V6li2mkCAwEAAaMjMCEwHwYDVR0RBBgwFoIIYXBpNi5jb22CCiouYXBpNi5j +b20wDQYJKoZIhvcNAQELBQADggIBAGmueLf8FFxVCgKKPQbHgPJ8/bTBQfBXi4cG +eg46QT3j4r8cw05CM64NXQs51q8kWa4aIhGzXweZPZZcnLppIZX31HguAsaOTWEv +2T3v9nqacMdujdyB+ll/bkwBoVudF47/JqQvHkevE0W43EZGgtmaPWki6psXK0C7 +Fwx95FeUizYrbsDDKbPKQ2iv70oES2bhhKi6K3o6uV6cFuUTxoNsjv9mjMM93xL/ +r5HhOAg7/voXwkvYAkOoehGRF4pwsfmjCJMF9k3BqDOZM+8087EHZFiWCPOvseA/ +FNxtEiDeUt86BGO9wzTv4ZN0gkJOrxATIw15wRemxnXJ8GUweiZh+U2nvDSklZq4 +Z4Wj2tWfa/yIqRBOZyqcAOS6jCnz/pYEGRniO6DMZSN1VX8A5pH8HNMnREW8Sze+ +c9cNZwquESqGFfKMAHOzuyxJhqEvuwwT/JNCLUgtQICPtdAQmNJQEwDAIdmS8VrX +XNsBIYSloIopKd7IY3V7Y8yASs7jKLrtJN4AE+SpssWuTa4p2LSO08sW3NP8r86G +1cs5R6Mckmqaqk5ES9gRbKmhm4goamb2fe2HJ/PTFyOrWtucA6OU3AdrQNXY+qbK +FskpOMlyl8WZo6JMsbOjd+tVygf9QuhvH9MxYDvppfeHxS0B7mYGZDOV/4G2vn2A +C1K1xkvo +-----END CERTIFICATE-----` + keyUpdate := `-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEAlXGEL4P5+mjllbb4VxkpqjsHny2wY7QZNbA0INfLdBq3B5av +L0Gp/nHk1iOMq+RkUu+DDb9i0wKw0j1ygaaN4WNXCa7/LZ7EIVDKcCKF7JQUi5za +9A36nTBMLb7ZbFwxnxGkCK2AwdIbUIB6Ciq/d+WEbZelspPQPtG9e8pVYTbNhRub +lGgVD+LEUHYdvSneveFQxdmHhNQMPXNbh+Qmgz48n7cLMEHhNwRyrWOb3zhdDeKl +Tm/yfPAQwvQNet+Ol3g19tKhS7WzDndrPfPNYAsoOdXZ5TTYAlXzwsbVYSOW4gMW +39p68ZnoseQHBVZkiTfN2CGvWbzd9gUXOiQship+xRfSoVEY5FN1KsTIA5fdDbgq +CTM4Ed1FTq39hJSBnsQObzQtfjgz0Sivwp9q0NoXX+YM6eBJ1kzVx6r70UMq+NJb +s7mvAW9Ehzv2m2GsinMq3lyvVwSbsJWX9vOHKxmuWympmOw17sUcH4DNHCxnzLK0 +L0oDV2atpNyKFvmMWSSmw09wOvuE+G+CLwILvgiSiTRPe1GmcgaTC92nZsafA5d8 +r2uxxJRHD7PAQSE5YnsdLsXAFSegYyHss2jq8lzVFwWhTk4tA/8si8qUHh/hHKt7 +1dYrSRmYTTyj834P8KDwQAMYBGcThtZa7PudOR22toohyte7Y7j4V6li2mkCAwEA +AQKCAgEAipLCQdUdHWfbS6JoUUtR6TnnbWDOaQV9Qt1q2PGBBN4oze6Z7zXyTbCK +w04fiNy1cnoJidvn5UZfc/Pqk/I/KboV4TLWCBVqRIJH1QcOLDt7eaIvHZNAyjUY +zmpj7ijnElsnQayw8gjDrzgS8g6FkMXQsFaoHRkXRsjx7THHTeelV0IsV8bTkPFz +nDCKjveeRXACmBOHqFGAMBMh0rZqR9SUHn168HqGPZ/wPntY8/mtv8xkAIGp1tQ8 +lqn7Pe7CqA2d7IuPaUbJDCcL2FyUGfT+jfKQpAsGKdRNvlTlU7fOlzDKKzTKP/G5 +ZgrNv9NGUj7erwU9Nfb90r0RtqYIac2CQnBDNx6snsSLx+QlO5wEQ8+xZ3zjKkGi +VazSaSqpkK2MV7KwyxJD5WEMqyHuUufQGU8TVywi4RkH21VXiwxwCbRLpF/gdBMi +2015JF1qd/t4GSHc6CtxqKslSxvNpr2cG6c/5Tx2gC3X6ARMr01YDFdKKI/FrsTQ +w3/E/WNKoPNgAXy361oaT5QVGOyBbL8h9mmDW6vD3R0aLLzdZKsQU+DNs6tcAl10 +mboKMBDVyg2Qew5QsumD89EpA3Da9VkyMxcPmOJ7oOJqscNF3Ejyx8ePlF+jS3dR +467uaXERVCGep2WDuYcv5Y14uUYAVj+P9aH85YjozA2KzG0aiqECggEBAMZvZwwd +fgub7RrU/0sHWw50JupVaRIjVIgopTmUw6MWDcpjn6BiCsMCJlabHxeN8XJYn3+E +aro8TZMvMYvjQQ0yrdKbPw8VLoTNfu++Bn/HPQ8gHFV+wzk33xfKRlRqxXic+MlG +SQ33IV+xr7JoW4fiMjSvOpp3kJj459mSOBLjhcXW7N4sMPhY6O72g2pqa8ncmOJT +ZU94VlssAxL3B1P1wsH8HsjhIluDHT+9qwsHhq/prIGLs4ydQSNB0m1oDT21zK/R +jC7fDbTT4oTZzy8QYiLDwW4ugct53HMQCnJfOdX4F6aOxt7k4d8VwoGuliKyjHii +VX4C+LZfT/64XiUCggEBAMDLyfqY8PK4ULNPrpSh92IOX5iwCcwGmOYDwGIeIbO6 +S1Pp7WGhA6Ryap/+F5gIdhB0qfnhAslMrQNRFfo3EcHF86UiCsfIRbPh7fwC2s0F +4G+tST6TsPyCddjWiQsvsmT1eYPNDgPj6JDl3d0mzboVsVge46RORTAhm4k2pobZ +EUDl+bljWM4LTf+pjN4DTRptwwueSqLVfxdkMhrHSG/NAYFerUKozxJGZeavnUcL +c+WUCtPJvyQrx2CBhn8LQ7xsPJJPYjNwSf6joMNXPyGvfPYQaxaK1NM0y/7KfuYN +N8DoBwwnpZIvcznjHDWY0P/cIZFKC2mmq6N062z/bfUCggEAey2oQAMGvVobgy55 +EzALvBsqFQjT4miADs18UxQfpVsJUHsrGboCiC8LcXN1h3+bQ6nzyIqAXf8VAKqp +DPcS6IhvEm9AY7J4YAPYKiZBjow1QPBj5kZ8FUaze+caZUiqMEbwwLCapMqlsutv +70WMm/szwzSLIlvaLLtF4O89U6xc3ASgoQG5nFBEuCHaTfKl2nbPiJ7QItbGdG4L +sngZ2mqSbSx+R6BJXZk0TN8GECCp4QUjCn+YA0+Sobo4T6Xpokb6OqHPbUEVFwz4 +bhNu4v4+jOoLZsQD2jVZPSvV8E1gb4xD0iaLGM3n0D2HskyX8g332OKcQ07A6SSd +WbdE6QKCAQEAiC2pzgNHdfowrmcjBkNdLHrAlWYKlX03dIjD08o6vethl7UNAj+s +BfT3UWk1myKm2jq9cQ25XRx2vHgC0Qki1r8OuN5RxQm2CjgUVERj7hsvi1JYAQZr +JgC0YuQuSqN3G460NR+ava62r9pdmv70o3L9ICQ5YO4UOsoSRZo/h9I9OJz4hjUh +HfCoOGS3Zn3ocTmEYml9iITKz2fraDTI+odQf+Oy9/mqwdrN0WLL8cmqJEgsWaoQ +A+mUW5tBt+zp/GZrZmECGRlAesdzH2c55X5CAsBYE8UeTMznJmI7vh0p+20oxTIf +5iDz/7hmTYlSXtdLMoedhhO++qb0P7owHQKCAQBbjTZSMRiqevr3JlEouxioZ+XD +yol9hMGQEquLWzqWlvP/LvgNT0PNRcE/qrMlPo7kzoq0r8PbL22tBl2C7C0e+BIv +UnBVSIGJ/c0AhVSDuOAJiF36pvsDysTZXMTFE/9i5bkGOiwtzRNe4Hym/SEZUCpn +4hL3iPXCYEWz5hDeoucgdZQjHv2JeQQ3NR9OokwQOwvzlx8uYwgV/6ZT9dJJ3AZL +8Z1U5/a5LhrDtIKAsCCpRx99P++Eqt2M2YV7jZcTfEbEvxP4XBYcdh30nbq1uEhs +4zEnK1pMx5PnEljN1mcgmL2TPsMVN5DN9zXHW5eNQ6wfXR8rCfHwVIVcUuaB +-----END RSA PRIVATE KEY-----` + keyCompareUpdate := "HrMHUvE9Esvn7GnZ+vAynaIg/8wlB3r0zm0htmnwofY0a95jf9O5bkBT8pEwjhLvcZOysVlRXE9fYFZ7heHoaihZmZIcnNPPi/SnNr1qVExgIWFYCf6QzpMdv7bMKag8AnYlalvbEIAyJA2tjZ0Gt9aQ9YlzmbGtyFX344481bSfLR/3fpNABO2j/6C6IQxxaGOPRiUeBEJ4VwPxmCUecRPWOHgQfyROReELWwkTIXZ17j0YeABDHWpsHASTjMdupvdwma20TlA3ruNV9WqDn1VE8hDTB4waAImqbZI0bBMdqDFVE0q50DSl2uzzO8X825CLjIa/E0U6JPid41hGOdadZph5Gbpnlou8xwOgRfzG1yyptPCKrAJcgIvsSz/CsYCqaoPCpil4TFjUq4PH0cWo6GlXN95TPX0LrAOh8WMCb7lZYXq5Q2TZ/sn5jF1GIiZZFWVUZujXK2og0I042 [...] + // key update compare + err = s.NewSecret(secretName, certUpdate, keyUpdate) + assert.Nil(ginkgo.GinkgoT(), err, "create secret error") + // check ssl in APISIX + time.Sleep(10 * time.Second) + tlsUpdate, err := s.ListApisixSsl() + assert.Nil(ginkgo.GinkgoT(), err, "list tlsUpdate error") + assert.Len(ginkgo.GinkgoT(), tlsUpdate, 1, "tls number not expect") + assert.Equal(ginkgo.GinkgoT(), certUpdate, tlsUpdate[0].Cert, "tls cert not expect") + assert.Equal(ginkgo.GinkgoT(), keyCompareUpdate, tlsUpdate[0].Key, "tls key not expect") + // check DP + s.NewAPISIXHttpsClient(host).GET("/ip").WithHeader("Host", host).Expect().Status(http.StatusOK).Body().Raw() + + // delete ApisixTls + err = s.DeleteApisixTls(tlsName, host, secretName) + assert.Nil(ginkgo.GinkgoT(), err, "delete tls error") + // check ssl in APISIX + time.Sleep(10 * time.Second) + tls, err = s.ListApisixSsl() + assert.Nil(ginkgo.GinkgoT(), err, "list tls error") + assert.Len(ginkgo.GinkgoT(), tls, 0, "tls number not expect") + }) }) diff --git a/test/e2e/scaffold/ssl.go b/test/e2e/scaffold/ssl.go index e96fa45..65531cd 100644 --- a/test/e2e/scaffold/ssl.go +++ b/test/e2e/scaffold/ssl.go @@ -32,6 +32,15 @@ data: cert: %s key: %s ` + _kubeTlsSecretTemplate = ` +apiVersion: v1 +kind: Secret +metadata: + name: %s +data: + tls.crt: %s + tls.key: %s +` _clientCASecretTemplate = ` apiVersion: v1 kind: Secret @@ -82,6 +91,17 @@ func (s *Scaffold) NewSecret(name, cert, key string) error { return nil } +// NewKubeTlsSecret new a kube style tls secret +func (s *Scaffold) NewKubeTlsSecret(name, cert, key string) error { + certBase64 := base64.StdEncoding.EncodeToString([]byte(cert)) + keyBase64 := base64.StdEncoding.EncodeToString([]byte(key)) + secret := fmt.Sprintf(_kubeTlsSecretTemplate, name, certBase64, keyBase64) + if err := k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, secret); err != nil { + return err + } + return nil +} + // NewClientCASecret new a k8s secret func (s *Scaffold) NewClientCASecret(name, cert, key string) error { certBase64 := base64.StdEncoding.EncodeToString([]byte(cert))
