This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
The following commit(s) were added to refs/heads/main by this push:
new d722db6b Support etcd client authentication (#331)
d722db6b is described below
commit d722db6bcd52554ef429961600d617d28d561ae5
Author: hailin0 <[email protected]>
AuthorDate: Fri Oct 20 19:48:16 2023 +0800
Support etcd client authentication (#331)
* Support etcd client authentication
* Set authentication parameters from startup command
---------
Co-authored-by: Gao Hongtao <[email protected]>
---
CHANGES.md | 6 +
banyand/metadata/client.go | 23 ++
banyand/metadata/schema/etcd.go | 53 ++++
docs/installation/cluster.md | 44 +++
go.mod | 2 +-
test/integration/etcd/client_suite_test.go | 31 +++
test/integration/etcd/client_test.go | 302 +++++++++++++++++++++
test/integration/etcd/testdata/ca.crt | 22 ++
.../etcd/testdata/client-clientusage.crt | 24 ++
.../etcd/testdata/client-clientusage.key.insecure | 27 ++
.../etcd/testdata/server-serverusage.crt | 24 ++
.../etcd/testdata/server-serverusage.key.insecure | 27 ++
test/integration/etcd/testdata/server.crt | 24 ++
test/integration/etcd/testdata/server.key.insecure | 27 ++
14 files changed, 635 insertions(+), 1 deletion(-)
diff --git a/CHANGES.md b/CHANGES.md
index fd2e3767..9e740579 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -2,6 +2,12 @@
Release Notes.
+## 0.6.0
+
+### Features
+
+- Support etcd client authentication.
+
## 0.5.0
### Features
diff --git a/banyand/metadata/client.go b/banyand/metadata/client.go
index 411b3c03..a687a3e2 100644
--- a/banyand/metadata/client.go
+++ b/banyand/metadata/client.go
@@ -39,6 +39,16 @@ const (
flagEtcdEndpointsName = "etcd-endpoints"
)
+const flagEtcdUsername = "etcd-username"
+
+const flagEtcdPassword = "etcd-password"
+
+const flagEtcdTLSCAFile = "etcd-tls-ca-file"
+
+const flagEtcdTLSCertFile = "etcd-tls-cert-file"
+
+const flagEtcdTLSKeyFile = "etcd-tls-key-file"
+
// NewClient returns a new metadata client.
func NewClient(forceRegisterNode bool) (Service, error) {
return &clientService{
@@ -51,6 +61,11 @@ type clientService struct {
schemaRegistry schema.Registry
closer *run.Closer
namespace string
+ etcdUsername string
+ etcdPassword string
+ etcdTLSCAFile string
+ etcdTLSCertFile string
+ etcdTLSKeyFile string
endpoints []string
forceRegisterNode bool
}
@@ -63,6 +78,11 @@ func (s *clientService) FlagSet() *run.FlagSet {
fs := run.NewFlagSet("metadata")
fs.StringVar(&s.namespace, "namespace", DefaultNamespace, "The
namespace of the metadata stored in etcd")
fs.StringArrayVar(&s.endpoints, flagEtcdEndpointsName,
[]string{"http://localhost:2379"}, "A comma-delimited list of etcd endpoints")
+ fs.StringVar(&s.etcdUsername, flagEtcdUsername, "", "A username of
etcd")
+ fs.StringVar(&s.etcdPassword, flagEtcdPassword, "", "A password of etcd
user")
+ fs.StringVar(&s.etcdTLSCAFile, flagEtcdTLSCAFile, "", "Trusted
certificate authority")
+ fs.StringVar(&s.etcdTLSCertFile, flagEtcdTLSCertFile, "", "Etcd client
certificate")
+ fs.StringVar(&s.etcdTLSKeyFile, flagEtcdTLSKeyFile, "", "Private key
for the etcd client certificate.")
return fs
}
@@ -78,6 +98,9 @@ func (s *clientService) PreRun(ctx context.Context) error {
s.schemaRegistry, err = schema.NewEtcdSchemaRegistry(
schema.Namespace(s.namespace),
schema.ConfigureServerEndpoints(s.endpoints),
+ schema.ConfigureEtcdUser(s.etcdUsername, s.etcdPassword),
+ schema.ConfigureEtcdTLSCAFile(s.etcdTLSCAFile),
+ schema.ConfigureEtcdTLSCertAndKey(s.etcdTLSCertFile,
s.etcdTLSKeyFile),
)
if err != nil {
return err
diff --git a/banyand/metadata/schema/etcd.go b/banyand/metadata/schema/etcd.go
index 3abb630a..213f02a5 100644
--- a/banyand/metadata/schema/etcd.go
+++ b/banyand/metadata/schema/etcd.go
@@ -19,12 +19,14 @@ package schema
import (
"context"
+ "crypto/tls"
"fmt"
"path"
"sync"
"time"
"github.com/pkg/errors"
+ "go.etcd.io/etcd/client/pkg/v3/transport"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"
@@ -69,6 +71,33 @@ func ConfigureServerEndpoints(url []string) RegistryOption {
}
}
+// ConfigureEtcdUser sets a username & password of the etcd.
+func ConfigureEtcdUser(username string, password string) RegistryOption {
+ return func(config *etcdSchemaRegistryConfig) {
+ if username != "" && password != "" {
+ config.username = username
+ config.password = password
+ }
+ }
+}
+
+// ConfigureEtcdTLSCAFile sets a trusted ca file of the etcd tls config.
+func ConfigureEtcdTLSCAFile(file string) RegistryOption {
+ return func(config *etcdSchemaRegistryConfig) {
+ config.tlsCAFile = file
+ }
+}
+
+// ConfigureEtcdTLSCertAndKey sets a cert & key of the etcd tls config.
+func ConfigureEtcdTLSCertAndKey(certFile string, keyFile string)
RegistryOption {
+ return func(config *etcdSchemaRegistryConfig) {
+ if certFile != "" && keyFile != "" {
+ config.tlsCertFile = certFile
+ config.tlsKeyFile = keyFile
+ }
+ }
+}
+
type etcdSchemaRegistry struct {
namespace string
client *clientv3.Client
@@ -80,6 +109,11 @@ type etcdSchemaRegistry struct {
type etcdSchemaRegistryConfig struct {
namespace string
+ username string
+ password string
+ tlsCAFile string
+ tlsCertFile string
+ tlsKeyFile string
serverEndpoints []string
}
@@ -135,6 +169,9 @@ func NewEtcdSchemaRegistry(options ...RegistryOption)
(Registry, error) {
DialKeepAliveTimeout: 10 * time.Second,
AutoSyncInterval: 5 * time.Minute,
Logger: l,
+ Username: registryConfig.username,
+ Password: registryConfig.password,
+ TLS: extractTLSConfig(registryConfig),
}
client, err := clientv3.New(config)
if err != nil {
@@ -399,3 +436,19 @@ func formatKey(entityPrefix string, metadata
*commonv1.Metadata) string {
listPrefixesForEntity(metadata.GetGroup(), entityPrefix),
metadata.GetName())
}
+
+func extractTLSConfig(cfg *etcdSchemaRegistryConfig) *tls.Config {
+ if cfg.tlsCAFile == "" && cfg.tlsCertFile == "" && cfg.tlsKeyFile == ""
{
+ return nil
+ }
+ tlsInfo := transport.TLSInfo{
+ TrustedCAFile: cfg.tlsCAFile,
+ CertFile: cfg.tlsCertFile,
+ KeyFile: cfg.tlsKeyFile,
+ }
+ tlsConfig, err := tlsInfo.ClientConfig()
+ if err != nil {
+ return nil
+ }
+ return tlsConfig
+}
diff --git a/docs/installation/cluster.md b/docs/installation/cluster.md
index fce7dc5f..02f48c75 100644
--- a/docs/installation/cluster.md
+++ b/docs/installation/cluster.md
@@ -28,3 +28,47 @@ The host is registered to the etcd cluster by the
`banyand-server` automatically
- `node-host-provider=hostname` : Default. The OS's hostname is registered as
the host part in the address.
- `node-host-provider=ip` : The OS's the first non-loopback active IP
address(IPv4) is registered as the host part in the address.
- `node-host-provider=flag` : `node-host` is registered as the host part in
the address.
+
+## Etcd Authentication
+
+`etcd` supports through tls certificates and RBAC-based authentication for
both clients to server communication. This section tends to help users set up
authentication for BanyanDB.
+
+### Authentication with username/password
+
+The etcd user can be setup by the [etcd authentication
guide](https://etcd.io/docs/v3.5/op-guide/authentication/)
+
+The username/password is configured in the following command:
+
+- `etcd-username`: The username for etcd client authentication.
+- `etcd-password`: The password for etcd client authentication.
+
+***Note: recommended using environment variables to set username/password for
higher security.***
+
+```shell
+$ ./banyand-server storage --etcd-endpoints=your-endpoints
--etcd-username=your-username --etcd-password=your-password <flags>
+$ ./banyand-server liaison --etcd-endpoints=your-endpoints
--etcd-username=your-username --etcd-password=your-password <flags>
+```
+
+### Transport security with HTTPS
+
+The etcd trusted certificate file can be setup by the [etcd transport security
model](https://etcd.io/docs/v3.5/op-guide/security/#example-1-client-to-server-transport-security-with-https)
+
+- `etcd-tls-ca-file`: The path of the trusted certificate file.
+
+```shell
+$ ./banyand-server storage --etcd-endpoints=your-https-endpoints
--etcd-tls-ca-file=youf-file-path <flags>
+$ ./banyand-server liaison --etcd-endpoints=your-https-endpoints
--etcd-tls-ca-file=youf-file-path <flags>
+```
+
+### Authentication with HTTPS client certificates
+
+The etcd client certificates can be setup by the [etcd transport security
model](https://etcd.io/docs/v3.5/op-guide/security/#example-2-client-to-server-authentication-with-https-client-certificates)
+
+- `etcd-tls-ca-file`: The path of the trusted certificate file.
+- `etcd-tls-cert-file`: Certificate used for SSL/TLS connections to etcd. When
this option is set, advertise-client-urls can use the HTTPS schema.
+- `etcd-tls-key-file`: Key for the certificate. Must be unencrypted.
+
+```shell
+$ ./banyand-server storage --etcd-endpoints=your-https-endpoints
--etcd-tls-ca-file=youf-file-path --etcd-tls-cert-file=youf-file-path
--etcd-tls-key-file=youf-file-path <flags>
+$ ./banyand-server liaison --etcd-endpoints=your-https-endpoints
--etcd-tls-ca-file=youf-file-path --etcd-tls-cert-file=youf-file-path
--etcd-tls-key-file=youf-file-path <flags>
+```
\ No newline at end of file
diff --git a/go.mod b/go.mod
index 10bd2daa..1dcf2b57 100644
--- a/go.mod
+++ b/go.mod
@@ -122,7 +122,7 @@ require (
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.etcd.io/bbolt v1.3.7 // indirect
go.etcd.io/etcd/api/v3 v3.5.9
- go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
+ go.etcd.io/etcd/client/pkg/v3 v3.5.9
go.etcd.io/etcd/client/v2 v2.305.9 // indirect
go.etcd.io/etcd/pkg/v3 v3.5.9 // indirect
go.etcd.io/etcd/raft/v3 v3.5.9 // indirect
diff --git a/test/integration/etcd/client_suite_test.go
b/test/integration/etcd/client_suite_test.go
new file mode 100644
index 00000000..23da32bf
--- /dev/null
+++ b/test/integration/etcd/client_suite_test.go
@@ -0,0 +1,31 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Package integration_setup_test is a integration test suite.
+package integration_etcd_test
+
+import (
+ "testing"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+func TestSetup(t *testing.T) {
+ RegisterFailHandler(Fail)
+ RunSpecs(t, "Client Suite")
+}
diff --git a/test/integration/etcd/client_test.go
b/test/integration/etcd/client_test.go
new file mode 100644
index 00000000..a78d8345
--- /dev/null
+++ b/test/integration/etcd/client_test.go
@@ -0,0 +1,302 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package integration_etcd_test
+
+import (
+ "context"
+ "crypto/tls"
+ "fmt"
+ "log"
+ "net/url"
+ "path/filepath"
+ "runtime"
+ "time"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ "github.com/onsi/gomega/gleak"
+ "go.etcd.io/etcd/client/pkg/v3/transport"
+ clientv3 "go.etcd.io/etcd/client/v3"
+ "go.etcd.io/etcd/server/v3/embed"
+
+ databasev1
"github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+ "github.com/apache/skywalking-banyandb/banyand/metadata/schema"
+ "github.com/apache/skywalking-banyandb/pkg/test"
+ "github.com/apache/skywalking-banyandb/pkg/test/flags"
+ "github.com/apache/skywalking-banyandb/pkg/test/helpers"
+ "github.com/apache/skywalking-banyandb/pkg/test/setup"
+)
+
+const host = "127.0.0.1"
+
+const namespace = "liaison-test"
+
+const nodeHost = "liaison-1"
+
+var _ = Describe("Client Test", func() {
+ var (
+ dir string
+ dirSpaceDef func()
+ caFilePath string
+ serverKeyFilePath string
+ serverCertFilePath string
+ clientKeyFilePath string
+ clientCertFilePath string
+ goods []gleak.Goroutine
+ )
+ BeforeEach(func() {
+ var err error
+ dir, dirSpaceDef, err = test.NewSpace()
+ Expect(err).ShouldNot(HaveOccurred())
+ _, currentFile, _, _ := runtime.Caller(0)
+ basePath := filepath.Dir(currentFile)
+ caFilePath = filepath.Join(basePath, "testdata/ca.crt")
+ serverKeyFilePath = filepath.Join(basePath,
"testdata/server-serverusage.key.insecure")
+ serverCertFilePath = filepath.Join(basePath,
"testdata/server-serverusage.crt")
+ clientKeyFilePath = filepath.Join(basePath,
"testdata/client-clientusage.key.insecure")
+ clientCertFilePath = filepath.Join(basePath,
"testdata/client-clientusage.crt")
+
+ goods = gleak.Goroutines()
+ })
+ AfterEach(func() {
+ dirSpaceDef()
+ Eventually(gleak.Goroutines,
flags.EventuallyTimeout).ShouldNot(gleak.HaveLeaked(goods))
+ })
+
+ It("should be using user/password connect etcd server successfully",
func() {
+ serverTLSInfo := transport.TLSInfo{}
+ clientTLSInfo := transport.TLSInfo{}
+ clientConfig, err := clientTLSInfo.ClientConfig()
+ Expect(err).ShouldNot(HaveOccurred())
+ username := "banyandb"
+ password := "banyandb"
+
+ etcdServer, err := createEtcdServer(dir, serverTLSInfo)
+ Expect(err).ShouldNot(HaveOccurred())
+ // wait for e.Server to join the cluster
+ <-etcdServer.Server.ReadyNotify()
+ etcdEndpoint := etcdServer.Config().ListenClientUrls[0].String()
+ defer etcdServer.Close()
+
+ adminClient, err := clientv3.New(clientv3.Config{
+ Endpoints: []string{etcdEndpoint},
+ })
+ Expect(err).ShouldNot(HaveOccurred())
+ defer adminClient.Close()
+ adminClient.RoleAdd(context.Background(), "root")
+ adminClient.UserAdd(context.Background(), "root", "root")
+ adminClient.UserGrantRole(context.Background(), "root", "root")
+ adminClient.UserAdd(context.Background(), username, password)
+ adminClient.UserGrantRole(context.Background(), username,
"root")
+ adminClient.AuthEnable(context.Background())
+
+ ports, err := test.AllocateFreePorts(2)
+ Expect(err).NotTo(HaveOccurred())
+ addr := fmt.Sprintf("%s:%d", host, ports[0])
+ httpAddr := fmt.Sprintf("%s:%d", host, ports[1])
+ closeFn := setup.CMD("liaison",
+ "--namespace", namespace,
+ "--grpc-host="+host,
+ fmt.Sprintf("--grpc-port=%d", ports[0]),
+ "--http-host="+host,
+ fmt.Sprintf("--http-port=%d", ports[1]),
+ "--http-grpc-addr="+addr,
+ "--node-host-provider", "flag",
+ "--node-host", nodeHost,
+ "--etcd-endpoints", etcdEndpoint,
+ "--etcd-username", username,
+ "--etcd-password", password)
+ defer closeFn()
+
+ Eventually(helpers.HTTPHealthCheck(httpAddr),
flags.EventuallyTimeout).Should(Succeed())
+ Eventually(func() (map[string]*databasev1.Node, error) {
+ return listKeys(etcdEndpoint, username, password,
clientConfig, fmt.Sprintf("/%s/nodes/%s:%d", namespace, nodeHost, ports[0]))
+ }, flags.EventuallyTimeout).Should(HaveLen(1))
+ })
+
+ It("should be using cacert connect etcd server successfully", func() {
+ serverTLSInfo := transport.TLSInfo{
+ KeyFile: serverKeyFilePath,
+ CertFile: serverCertFilePath,
+ }
+ clientTLSInfo := transport.TLSInfo{
+ TrustedCAFile: caFilePath,
+ }
+ clientConfig, err := clientTLSInfo.ClientConfig()
+ Expect(err).ShouldNot(HaveOccurred())
+
+ etcdServer, err := createEtcdServer(dir, serverTLSInfo)
+ Expect(err).ShouldNot(HaveOccurred())
+ // wait for e.Server to join the cluster
+ <-etcdServer.Server.ReadyNotify()
+ etcdEndpoint := etcdServer.Config().ListenClientUrls[0].String()
+ defer etcdServer.Close()
+
+ ports, err := test.AllocateFreePorts(2)
+ Expect(err).NotTo(HaveOccurred())
+ addr := fmt.Sprintf("%s:%d", host, ports[0])
+ httpAddr := fmt.Sprintf("%s:%d", host, ports[1])
+ closeFn := setup.CMD("liaison",
+ "--namespace", namespace,
+ "--grpc-host="+host,
+ fmt.Sprintf("--grpc-port=%d", ports[0]),
+ "--http-host="+host,
+ fmt.Sprintf("--http-port=%d", ports[1]),
+ "--http-grpc-addr="+addr,
+ "--node-host-provider", "flag",
+ "--node-host", nodeHost,
+ "--etcd-endpoints", etcdEndpoint,
+ "--etcd-tls-ca-file", caFilePath)
+ defer closeFn()
+
+ Eventually(helpers.HTTPHealthCheck(httpAddr),
flags.EventuallyTimeout).Should(Succeed())
+ Eventually(func() (map[string]*databasev1.Node, error) {
+ return listKeys(etcdEndpoint, "", "", clientConfig,
fmt.Sprintf("/%s/nodes/%s:%d", namespace, nodeHost, ports[0]))
+ }, flags.EventuallyTimeout).Should(HaveLen(1))
+ })
+
+ It("should be using key pair connect etcd server successfully", func() {
+ serverTLSInfo := transport.TLSInfo{
+ KeyFile: serverKeyFilePath,
+ CertFile: serverCertFilePath,
+ TrustedCAFile: caFilePath,
+ ClientCertAuth: true,
+ }
+ clientTLSInfo := transport.TLSInfo{
+ KeyFile: clientKeyFilePath,
+ CertFile: clientCertFilePath,
+ TrustedCAFile: caFilePath,
+ }
+ clientConfig, err := clientTLSInfo.ClientConfig()
+ Expect(err).ShouldNot(HaveOccurred())
+
+ etcdServer, err := createEtcdServer(dir, serverTLSInfo)
+ Expect(err).ShouldNot(HaveOccurred())
+ // wait for e.Server to join the cluster
+ <-etcdServer.Server.ReadyNotify()
+ etcdEndpoint := etcdServer.Config().ListenClientUrls[0].String()
+ defer etcdServer.Close()
+
+ ports, err := test.AllocateFreePorts(2)
+ Expect(err).NotTo(HaveOccurred())
+ addr := fmt.Sprintf("%s:%d", host, ports[0])
+ httpAddr := fmt.Sprintf("%s:%d", host, ports[1])
+ closeFn := setup.CMD("liaison",
+ "--namespace", namespace,
+ "--grpc-host="+host,
+ fmt.Sprintf("--grpc-port=%d", ports[0]),
+ "--http-host="+host,
+ fmt.Sprintf("--http-port=%d", ports[1]),
+ "--http-grpc-addr="+addr,
+ "--node-host-provider", "flag",
+ "--node-host", nodeHost,
+ "--etcd-endpoints", etcdEndpoint,
+ "--etcd-tls-ca-file", caFilePath,
+ "--etcd-tls-cert-file", clientCertFilePath,
+ "--etcd-tls-key-file", clientKeyFilePath)
+ defer closeFn()
+
+ Eventually(helpers.HTTPHealthCheck(httpAddr),
flags.EventuallyTimeout).Should(Succeed())
+ Eventually(func() (map[string]*databasev1.Node, error) {
+ return listKeys(etcdEndpoint, "", "", clientConfig,
fmt.Sprintf("/%s/nodes/%s:%d", namespace, nodeHost, ports[0]))
+ }, flags.EventuallyTimeout).Should(HaveLen(1))
+ })
+})
+
+func parseURLs(urls []string) ([]url.URL, error) {
+ uu := make([]url.URL, 0, len(urls))
+ for _, u := range urls {
+ cURL, err := url.Parse(u)
+ if err != nil {
+ return nil, err
+ }
+ uu = append(uu, *cURL)
+ }
+ return uu, nil
+}
+
+func createEtcdServer(dir string, tlsInfo transport.TLSInfo) (e *embed.Etcd,
err error) {
+ ports, err := test.AllocateFreePorts(2)
+ if err != nil {
+ return nil, err
+ }
+ enableTLS := tlsInfo.TrustedCAFile != "" || tlsInfo.CertFile != "" ||
tlsInfo.KeyFile != ""
+ httpProtocol := "http"
+ if enableTLS {
+ httpProtocol = "https"
+ }
+ clientURL := fmt.Sprintf("%s://%s:%d", httpProtocol, host, ports[0])
+ peerURL := fmt.Sprintf("%s://%s:%d", httpProtocol, host, ports[1])
+ cURLs, err := parseURLs([]string{clientURL})
+ if err != nil {
+ return nil, err
+ }
+ pURLs, err := parseURLs([]string{peerURL})
+ if err != nil {
+ return nil, err
+ }
+
+ etcdConfig := embed.NewConfig()
+ etcdConfig.Name = "test-etcd-certificate"
+ etcdConfig.Dir = dir
+ etcdConfig.ListenClientUrls, etcdConfig.AdvertiseClientUrls = cURLs,
cURLs
+ etcdConfig.ListenPeerUrls, etcdConfig.AdvertisePeerUrls = pURLs, pURLs
+ etcdConfig.InitialCluster =
etcdConfig.InitialClusterFromName(etcdConfig.Name)
+ etcdConfig.ClientTLSInfo = tlsInfo
+ etcdConfig.PeerTLSInfo = tlsInfo
+
+ return embed.StartEtcd(etcdConfig)
+}
+
+func listKeys(serverAddress string, username string, password string, tls
*tls.Config, prefix string) (map[string]*databasev1.Node, error) {
+ defaultDialTimeout := 5 * time.Second
+ client, err := clientv3.New(clientv3.Config{
+ Endpoints: []string{serverAddress},
+ TLS: tls,
+ Username: username,
+ Password: password,
+ DialTimeout: defaultDialTimeout,
+ })
+ if err != nil {
+ log.Fatalf("Failed to create etcd client: %v", err)
+ }
+ defer client.Close()
+
+ ctx, cancel := context.WithTimeout(context.Background(),
defaultDialTimeout)
+ defer cancel()
+
+ resp, err := client.Get(ctx, prefix, clientv3.WithPrefix())
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.Count == 0 {
+ return nil, nil
+ }
+
+ nodeMap := make(map[string]*databasev1.Node)
+ for _, kv := range resp.Kvs {
+ md, err := schema.KindNode.Unmarshal(kv)
+ if err != nil {
+ return nil, err
+ }
+ nodeMap[string(kv.Key)] = md.Spec.(*databasev1.Node)
+ }
+
+ return nodeMap, nil
+}
diff --git a/test/integration/etcd/testdata/ca.crt
b/test/integration/etcd/testdata/ca.crt
new file mode 100644
index 00000000..8e373720
--- /dev/null
+++ b/test/integration/etcd/testdata/ca.crt
@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDrjCCApagAwIBAgIUNkN+TZ3hgHno+H9j56nWkmb4dBEwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0yMTAyMjgxMDQ4MDBaFw0zMTAyMjYxMDQ4
+MDBaMG8xDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTELMAkGA1UEAxMCY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQDZwQPFZB+Kt6RIzYvTgbNlRIX/cLVknIy4ZqhLYDQNOdosJn04jjkCfS3k
+F5JZuabkUs6d6JcLTbLWV5hCrwZVlCFf3PDn6DvK12GZpybhuqMPZ2T8P2U17AFP
+mUj/Rm+25t8Er5r+8ijZmqVi1X1Ef041CFGESr3KjaMjec2kYf38cfEOp2Yq1JWO
+0wpVfLElnyDQY9XILdnBepCRZYPq1eW1OSkRk+dZQnJP6BO95IoyREDuBUeTrteR
+7dHHTF9AAgR5tnyZ+eLuVUZ2kskcWLxH3y9RyjvVJ+1uCzbdydVPf0H1pBoqWcuA
+PYjYkLKMOKBWfYJhSzykhf+QMC7xAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
+BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQpJiv07dkY9WB0zgB6wOb/HMi8oDAN
+BgkqhkiG9w0BAQsFAAOCAQEA0TQ8rRmLt4wYjz0BKh+jElMIg6LBPsCPpfmGDLmK
+fdj4Jp7QFlLmXlQSmm8zKz3ftKoOFPYGQYHUkIirIrQB/tdHoXJLgxCzI0SrdCiM
+m/DPVjfOTa9Mm5rPcUR79rGDLj2BgzDB+NTETVDXo8mAL5MjFdUyh6jOGBctkCG/
+TWdUaN33ZLwUl488NLaw98fIZ/F4d/dsyCJvHEaoo++dgjduoQxmH9Scr2Frmd8G
+zYxOoZHG3ARBDp2mpr+I3UCR1/KTITF/NXL6gDcNY3wyZzoaGua7Bd/ysMSi1w3j
+CyvClSvRPJRLQemGUP7B/Y8FUkbJ2i/7tz6ozn8sLi3V2Q==
+-----END CERTIFICATE-----
diff --git a/test/integration/etcd/testdata/client-clientusage.crt
b/test/integration/etcd/testdata/client-clientusage.crt
new file mode 100644
index 00000000..71b305fb
--- /dev/null
+++ b/test/integration/etcd/testdata/client-clientusage.crt
@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIECDCCAvCgAwIBAgIULbzkAv8zbkJzZIRDPnBwXl0/BH0wDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0yMTAyMjgxMDQ4MDBaFw0zMTAyMjYxMDQ4
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDWBNo9tYRoQKv76xabz0EPXGJKHIrUjf0NbXz3d9jbP2sH
+3hutXr/A221pULfZYIZdaUtmEuEr1905nYwJ2gnO9Y/iSc6fQ/4EjoT+VZLdINQw
+I1dG2rtv2ZuYL5oYfgCjLkV1LzYuyfY/zJ93WoJW0YA0t50MEQNGEqD7pYlhsPej
+iGyjagSi7zsoAkAagNprULH6RyAqDG7db+MfJOUzHUv4PWGBXPb0PHY3xA+WayFB
+nP5AZO16oDh/UnzvfEAJULXeIOLs4eOmtzKMwZwrWzgCB+jBeVlc1FOwXQcmBamN
+eYUs75GoO9aSSLROvnQiw2P0z0xVNmDokDXGsSRxAgMBAAGjgZIwgY8wDgYDVR0P
+AQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYD
+VR0OBBYEFCB4ysDF81d6lkKIvebj08BcRWNoMB8GA1UdIwQYMBaAFCkmK/Tt2Rj1
+YHTOAHrA5v8cyLygMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG
+9w0BAQsFAAOCAQEAo2B+piCBTjdpCLFj/kc+A0alZTbNdr0+BTsN+5aBE9k4JlZS
+smkIQL0vyzjKw/W/o2EyPVcVKJX52/GQsC3bQrBb2lH1jRYgt5pRo24kKHy4Nlc3
+IaYg++ssfT2ZdpYiL3lzLyOHEumcynz3nI5M81e5CCIdEennxaM8FuiYN5OXDOR3
+j+bCYHLYPaWYZopfiSrnq+Z4gRUS2sMI1yqtiPSUdIJLnTfyEEdexvs/KUtFWvFO
+4AcecKvT6HA8oNDiWfE6e854uDLTkbXW1rK+FWPU9pv5NR50+GBCvxvmDGtGXxQu
+yu+kOsx2gfgNc4idIv1pjZF/1YzrrKGAhChN2A==
+-----END CERTIFICATE-----
diff --git a/test/integration/etcd/testdata/client-clientusage.key.insecure
b/test/integration/etcd/testdata/client-clientusage.key.insecure
new file mode 100644
index 00000000..ea139257
--- /dev/null
+++ b/test/integration/etcd/testdata/client-clientusage.key.insecure
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA1gTaPbWEaECr++sWm89BD1xiShyK1I39DW1893fY2z9rB94b
+rV6/wNttaVC32WCGXWlLZhLhK9fdOZ2MCdoJzvWP4knOn0P+BI6E/lWS3SDUMCNX
+Rtq7b9mbmC+aGH4Aoy5FdS82Lsn2P8yfd1qCVtGANLedDBEDRhKg+6WJYbD3o4hs
+o2oEou87KAJAGoDaa1Cx+kcgKgxu3W/jHyTlMx1L+D1hgVz29Dx2N8QPlmshQZz+
+QGTteqA4f1J873xACVC13iDi7OHjprcyjMGcK1s4AgfowXlZXNRTsF0HJgWpjXmF
+LO+RqDvWkki0Tr50IsNj9M9MVTZg6JA1xrEkcQIDAQABAoIBAAGBZTub5EOLeOo7
+vBv6eD2wa6yTyNI38Xi/tWpUOH1KU+lpQY6VpQmpQXrFK5Xm3OsZS4N7TIQvb4nx
+NsP2+aywA4QW+tIZ+1Zy3jKfzXmqunNgPEPuU/U0dai7ZP0ZHc4IDEsHuvzXRNks
+Ck8fnt0XeixkwkEMeZZrmSBMCMxcHAWxiv+oXF+olN3vTD2aDC8T6YwahMyQUQfW
+IA9fuO8Dzzmk2I7mDHa29cbB+PW4E5tkJmHVZqEu8jPgMjCJGc2IR1YpLAXF8YBB
+vgh6ZgI6JOg1OiNETuQekamAMOblFVOdPUjPSxuyJzEE8VpIdD3Z9UMNq+FDQh/F
+j1lEEEECgYEA9nYwUh+e0H9c9IRBLNYAbq2PV4SpFKvFrHOTQpylMPisUTgdHKLT
+CvO1wbNprElBAulOWobCyKshWGd5ECFsCvsWS6xmGi442q3ov5xtAMmvSmtW8s+8
+tUeVRQGS/Yn5Uxj2msUPe6vJEniLgsxmbFbDYqvr65COrAsCDEY3DkkCgYEA3k09
+EGhiO1joDtJPI21vUzzecBuep32oKiwip3OgS/mct04/QR+6lp1x4sPMYlyxbyk9
+jPdkzU07d8r+mES9RweE5lc1aCaF5eA8y6qtL9vBgsXRiEXlpYLxb0TOQaYNU0qM
+aYumYPWjsjwYDvRKaVzThFUkYwapKFqtMV98BOkCgYAkIOkucLIwMCtpMKX5M5m2
+n7yegLTkcdW1VO/mWN4iUqG3+jjSRNAZD+a58VnxRn/ANIEm5hBRqDxoICrwAWY8
+Kdh32VrSRapR7CJtTDnyXp5Sk2+YgnlQPaEVD4kDn6Er3EHyKCb/4wvDqGYTE3GE
+OifEJB2eV3+Cms5/DB/v+QKBgFzV8r9saEGSkm7GI2iPJiOj0t0Mm8gksNrTzbES
+l4nC91CR69adkoWdwNbLoAof3bWnil3ZXw5hx4jyjDo40rbcDANJvjL9i4OBjsIb
+R/Ipmvmq9SMs1Ye2VG98U4qU9xGmm1bkjBoH21HuyLlOCdlQe8DS8bwtJu2EWLm6
+v4cpAoGAP3pqi6iIZAbJVqw6p5Xd/BLzn3iS+lwQjwF/IWp5UnFCGovnVSJG2wqP
+kxL9jy4asMDDuMKzEzO3+UT6WBRI+idV8PgDNEYkXcnVAA5lZ+2kCJwRICsC6MYH
+1nIHJtPngUrwT3TUhMp/WfpYUjTdiOC3aJmKq/NGZxE8/Sb3G6U=
+-----END RSA PRIVATE KEY-----
diff --git a/test/integration/etcd/testdata/server-serverusage.crt
b/test/integration/etcd/testdata/server-serverusage.crt
new file mode 100644
index 00000000..b7aca3e5
--- /dev/null
+++ b/test/integration/etcd/testdata/server-serverusage.crt
@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIECDCCAvCgAwIBAgIUDVfELNb4NV52PJjENU2DNOIFx6IwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0yMTAyMjgxMDQ4MDBaFw0zMTAyMjYxMDQ4
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDvx5mE6ZwV61si7QK18X+K6lJSWr8rE+0l4YbvbLh/bCdP
+bnPzTsiK+FN4FA9fur73L/RvQDxO9XG99pKIhKPGxiit6DSJ9rL24RpK7tJpgnnK
+Kdry6E+6/ZvVXBP3LtdJyX3kpdw+TSivlwi50CeEQA5argOXnyDNGIOWC8iMpeg7
+z3pE0DfpDEgLMjGE/I/9YHRCOiK/8kQchXqVfWHpALFakf9+QPNpNrEIPjV7IPja
+GEUeDG9MI0PYbBl45NkKIi0nelV/Nay9kyPSPKfvngJU2dTqsGSVXW+DlZo7OJBh
+RLEih+CWGfotGzFzXWdQRvD/VleaxYYUDKkyHVx1AgMBAAGjgZIwgY8wDgYDVR0P
+AQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYD
+VR0OBBYEFKdzgCE8K0tYt9LaF5N2aQrrHru0MB8GA1UdIwQYMBaAFCkmK/Tt2Rj1
+YHTOAHrA5v8cyLygMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG
+9w0BAQsFAAOCAQEAYcD+1ebiJna7dfydgw/yox+b6/KO16evaU4c5Spu5O+VtcZG
+rKMi8MT8V0U7kL9Xo9TszbzwpPWr1It0wcmM1cZrwykkT/baVJADaLtfSFUtlCDX
+HNB04C2UUBPPosFr1d5YtwyN55qxgyMg+IDeMubYZ4qwDWCYBTIiz9yHoQ6LuuV8
+Tkkpa6X5n4+fO2iUgA6SZUkwZGdbQLOz9VMa1qgyOz3ejuDeMc4sa08iADs4wG/X
+ohRGg0Df5THeXhR+Pn0HBf3T0eTAeZzLL5xtlIn9o6o9CEU573uEYQI1BG1kcDeQ
+Rs9J/2iuLqr8GAjr7k5aUW3FFYRtqC3YR0G5Eg==
+-----END CERTIFICATE-----
diff --git a/test/integration/etcd/testdata/server-serverusage.key.insecure
b/test/integration/etcd/testdata/server-serverusage.key.insecure
new file mode 100644
index 00000000..a0ac946a
--- /dev/null
+++ b/test/integration/etcd/testdata/server-serverusage.key.insecure
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEA78eZhOmcFetbIu0CtfF/iupSUlq/KxPtJeGG72y4f2wnT25z
+807IivhTeBQPX7q+9y/0b0A8TvVxvfaSiISjxsYoreg0ifay9uEaSu7SaYJ5yina
+8uhPuv2b1VwT9y7XScl95KXcPk0or5cIudAnhEAOWq4Dl58gzRiDlgvIjKXoO896
+RNA36QxICzIxhPyP/WB0Qjoiv/JEHIV6lX1h6QCxWpH/fkDzaTaxCD41eyD42hhF
+HgxvTCND2GwZeOTZCiItJ3pVfzWsvZMj0jyn754CVNnU6rBklV1vg5WaOziQYUSx
+Iofglhn6LRsxc11nUEbw/1ZXmsWGFAypMh1cdQIDAQABAoIBAQCnpgkisyuc78fy
+7YAdslKY0Cjqx+QtvGrtN3he4sdE4FvD39hWX9k7wVCq/muZZTqsHe1r85+3HUl/
+pmzh4suX6Wj73wUNCV4r20vE5KJdfwqkXQtnFyLX/QX98blL9IY2YxkQyx7ouI4f
+5xwEvxNCFn9yy4RbeLk4bVFjka2RF/x6qEUCHq5Q74vWvyC1i3kGKgYruM39RQw3
+D5fG8xdUexBc32nfzynP+0NcFAiy+yUQWOLcE4i8XaegFvg+QvWOx1iwjqU3FDeC
+JzKrtw9SLBWf7AGraxA59K4WJ63xqGqFugWcFaYh923X8zES/s0wrtV2T14Lgj3Q
+aWJ0DfQBAoGBAPNd1Aph0Z9PwZ1zivJprj8XDnFP2XIqlODYuiVddjbvIV13d6F/
+PE/ViW0MVduF0ejkPs9+hSxYOH58EWIt44Li/Nre1U42ny+fJrY1P5Mq5nriM4L4
+lx2YFaWzAoxzpMbbQ14kEMcQSicziDbBx62aaQYu4UwrvqXYdSYp+D+BAoGBAPw6
+Gtv6hytg19GtH6sQG9/4K4cLGX4lJE3pTL3eUicfoEI+hviZdG8FS77Uv05ga6fQ
+OlyqvpmmXp6fgTrSlHBeKO75A3sT7fP1v1foq1y+CdMGytOnJENUc80bN0L1dFI1
+zwYm7eLDP0KdUYpf+Rpgcap4StQbotpc6oy705b1AoGBAO9z26VXd+ybifKE9Cru
+Zp727aP6IAaf9RqCxCztl9oXUanoWVISoeIfRfeA0p2LPu06Xr7ESv5F01hIdMY4
+RonLE2W7KP+q6NfvbSSMogAIjvxLwslUFUPuFyaRSqmtQ2zR4qgnLkbfNUb7AkR2
+SCT9L+cAi3bp98ywfRvO4c6BAoGANkAJJudry1i5EtA5z4FXfYTTV+h7QzaZ6GgV
+qYD4CpIy1gy82xumf3qUICeCPkle3mlbJDNVa5btIxELqqtAYiregwfsR7yxoZdp
+4G6a7Qey9UCwv3Vjx1eS0LrZ1/0TV9ta++fDotJ7+Mf9kdWyromv6QqWjaikDnON
+v1dm20ECgYEA6i+uvBuomUzqJYjUTNgMtmfYjuXv8D58wLkY10si7T2ayAlFKBG2
+Dno/dojOcixO1dtaq7dA+KNXtESekjT1Y4VleGHWpEglRScE629iow4ASrluP/Ec
+F2DvTRW4daFDWQV4je1u0+wDj5B8KZjO/e759BztiRyRqTCzpxTa8Ms=
+-----END RSA PRIVATE KEY-----
diff --git a/test/integration/etcd/testdata/server.crt
b/test/integration/etcd/testdata/server.crt
new file mode 100644
index 00000000..04c3d7b6
--- /dev/null
+++ b/test/integration/etcd/testdata/server.crt
@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEEjCCAvqgAwIBAgIUPMH88Ekdi6JPq+5703+qGFH7VmMwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0yMTAyMjgxMDQ4MDBaFw0zMTAyMjYxMDQ4
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDD3vhO7LT77scWKnsozEg1DiQQsAbgGFfAoQJOvgrRv7V7
+I5+9n7hlpqKEIYkOuX0LSqpLBJ+9ORxXPBNZFKsytryOc3ZWTAoozUkufUOKfUFa
+1QpExA5u/FpwtNXGRGC35wus/JVTtcIiifeml2PIdyoxdXfev6y4yJvhO38Osqru
+GoySDORGsPrmLdpoUieofhmHWEgONpoY3fVsqAwiP1NMDNuqbVHvDjMykxj9AmQa
+WBdspsRXcgcl5Dp7mf1KVPRbvnCOjLuDDiBCwTTgpU3sDFyhHm/Bq29lPwlHWi7Z
+WKQYGIwqQOfOjfjhH009/+z11Z/1ovj+FWLbcXP9AgMBAAGjgZwwgZkwDgYDVR0P
+AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
+Af8EAjAAMB0GA1UdDgQWBBT0sUW5xBildDtaySEQYE9vX+8SZDAfBgNVHSMEGDAW
+gBQpJiv07dkY9WB0zgB6wOb/HMi8oDAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A
+AAEwDQYJKoZIhvcNAQELBQADggEBAATMlUra5N3Z4KB++xmGM6h9OhYmbKqn6IEV
+o4mqrTimyzgtvVsHh/v3XvBzxAdma6QjkZygfg+EIHSLbJVmZzxzr3YeENu4EyDc
+l+FfNPCyFHX9CH2Rk1ZThkQbmqVrzInXmG47G/PbTC2l8+kAZwvp37QfIJNCYIku
+XTp9R72sEkfNXxZxsZjwM7Z++LaB+cVEuLNJG0OpMhouTuxoN6pemzBmmFBP2mOr
+SeZnuEVtvDIbklJDdgcB/mPd7FE2xCVfa+p5Ol9Fcw5aPxTXAAQ+aVDtzh7jcFWk
+SV4K/ZYFqYN4E4H1UXlkHKj/qCmryvPxe8DVzu6Pd0ZyA0H4Lxk=
+-----END CERTIFICATE-----
diff --git a/test/integration/etcd/testdata/server.key.insecure
b/test/integration/etcd/testdata/server.key.insecure
new file mode 100644
index 00000000..b52bdc79
--- /dev/null
+++ b/test/integration/etcd/testdata/server.key.insecure
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAw974Tuy0++7HFip7KMxINQ4kELAG4BhXwKECTr4K0b+1eyOf
+vZ+4ZaaihCGJDrl9C0qqSwSfvTkcVzwTWRSrMra8jnN2VkwKKM1JLn1Din1BWtUK
+RMQObvxacLTVxkRgt+cLrPyVU7XCIon3ppdjyHcqMXV33r+suMib4Tt/DrKq7hqM
+kgzkRrD65i3aaFInqH4Zh1hIDjaaGN31bKgMIj9TTAzbqm1R7w4zMpMY/QJkGlgX
+bKbEV3IHJeQ6e5n9SlT0W75wjoy7gw4gQsE04KVN7AxcoR5vwatvZT8JR1ou2Vik
+GBiMKkDnzo344R9NPf/s9dWf9aL4/hVi23Fz/QIDAQABAoIBAQCKvee5UCYqxkoz
+Q0gV8A29txSI1YcpOVT/V41g5XCYfmk4nlVKZlahelVnrrF8wpr2Yp8ZoF7eFBQl
+HqK92Mwjkhkh9lt+aUJRAIiz63rqICspAfrSFuX6a7pMV2uNk2XHHlvA3vGPaBHp
+kTzgvh+qIe67NfAA0liwUzlHY3NunpXW6UQm2OWtabOfc1zZ78E58It0VTaAWKZq
+KDOxQGdwS4BGIUbsCvktPgncDDzi8wIZY/6YDTXkYKUJWqVEWf3SregXbX6/cjQa
+x8v2v95LiMk0vp9E5GG9QKdTEC+fZyQsq983G9t4O9VPw2JBR2TUxIRrPH0gkvhQ
+F1n3yYABAoGBAM1qHX1wQoHiOpVsgaXrAX2QWrBeNpbGAgVh67MsD+FlUH/ZikcK
+dsz/pTg+TmUuURKadXWJx43E/IVb6Bw8io0uF30aXeWRSIDK99FAmx2GxxTBge5f
+MtAQSTctr//yYLaNZWSdTpmaQYRtenK8zN6OTQ251M0sWyWsv3UcpBABAoGBAPQb
+NFqaw8C8JgJtHa7wrYCU9ShNxzQzo/jgu5ZuO6CxKNYyFGR8ZOpWQx8vTrNkjDXN
+iq9oZ3gIcm7c4TQOg3ydISNyoAEYrhRgQC6+rDkl4Zgby6ssvDHMqCclMp5Ztn3Y
+bfXG2gk3ULLH8TkQ2KWRJrpPT3iSdvLVXmNHHaP9AoGADAxhVm4zOHMQhJssr5Kt
+L7Q73YRpJ0bN74rizEuVUt8ibZ1Q4wHWHggQpM/iwUSKNNEiepZuQf5/4UKWxrE2
+XzmI3ymgwEpZOlStXHSxpHW3T5xaBqVG0bVi1f20CQsqaQq6G8CuT4wgs6fIOtqg
+GZ23H0r7FF25qugLAs9/QAECgYBsi6ROHb+p9oAYWBj474DXSmVxVJSd+9CQHK6N
+h9rv65czF/XFcSMWqOET/t9KGg3W5t0ifpRz5Z2s+n8RvNpvERfpQVEw656M5Pfl
+UVgX2WZlUwbPyQauRkkHjxzhGRdzAkhzH8dYjcZOmWYEcB9GEDNeaWH3RXmrJYHh
+N4BQqQKBgQCtdULs1ju0Y5zwSdkxhBWJHxCpLjuCpxfpWmnVFlp/Fj9S7rBfzWrt
+hYjsNyfMUm13jg4+6CJNdf3pQbdczXWTt/E0t/uoo1m8Azcd1qFPKV8wVOSiZBXv
+mMWHW8fw55ygoFyBlQpzwI6M+Zpwv3Smli/H9gQClsS2RmdIjUhqtg==
+-----END RSA PRIVATE KEY-----
