ocket8888 commented on a change in pull request #5924: URL: https://github.com/apache/trafficcontrol/pull/5924#discussion_r652056597
########## File path: docs/source/tools/traffic_vault_migrate.rst ########## @@ -0,0 +1,112 @@ +.. +.. +.. Licensed under the Apache License, Version 2.0 (the "License"); +.. you may not use this file except in compliance with the License. +.. You may obtain a copy of the License at +.. +.. http://www.apache.org/licenses/LICENSE-2.0 +.. +.. Unless required by applicable law or agreed to in writing, software +.. distributed under the License is distributed on an "AS IS" BASIS, +.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.. See the License for the specific language governing permissions and +.. limitations under the License. +.. + +.. _traffic_vault_migrate: + +######################### +Traffic Vault Migrate +######################### +The ``traffic_vault_migrate`` tool - located at :file:`tools/traffic_vault_migrate/traffic_vault_migrate.go` in the `Apache Traffic Control repository <https://github.com/apache/trafficcontrol>`_ - +is used to transfer TV keys between database servers. It interfaces directly with each backend so Traffic Ops/Vault being available is not a requirement. +The tool assumes that the schema for each backend is already setup as according to the :ref:`admin setup <traffic_vault_admin>`. + +.. program:: traffic_vault_migrate + +Usage +=========== +``traffic_vault_migrate [-cdhmr] [-f value] [-g value] [-o value] [-t value]`` + +.. option:: -c, --compare + + Compare 'to' and 'from' backend keys. Will fetch keys from the dbs of both 'to' and 'from', sorts them by cdn/ds/version and does a deep comparison. + +.. option:: -d, --dump + + Write keys (from 'from' server) to disk in the folder 'dump' with the unix permissions 0640. + + .. warning:: This can write potentially sensitive information to disk, use with care. + +.. option:: -f, --fromCfg=CFG + + From server config file (default "riak.json") + +.. option:: -g, --toCfg=CFG + + To server config file (default "pg.json") + +.. option:: -h, --help + + Displays usage information + +.. option:: -o, --toType=TYPE + + From server types (Riak|PG) (default "PG") + +.. option:: -m, --noConfirm + + Do not require confirmation before inserting records + +.. option:: -r, --dry + + Do not perform writes. Will do a basic output of the keys on the 'from' backend. + +.. option:: -t, --fromType=TYPE Review comment: options that take option-arguments and have long and short forms should specify it for both forms e.g. ```rst .. option:: -t TYPE, --from-type TYPE ``` The <kbd>=</kbd> can't be used with the short form, but using a space in the short form and an <kbd>=</kbd> in the long form doesn't hurt anything afaik. ########## File path: tools/traffic_vault_migrate/postgres.go ########## @@ -0,0 +1,742 @@ +package main + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + util "github.com/apache/trafficcontrol/lib/go-util" + + _ "github.com/lib/pq" +) + +// PGConfig represents the configuration options available to the PG backend +type PGConfig struct { + Host string `json:"host"` + Port string `json:"port"` + User string `json:"user"` + Password string `json:"password"` + SSLMode string `json:"sslmode"` + Database string `json:"database"` + Key string `json:"aesKey"` + AESKey []byte +} + +// PGBackend is the Postgres implementation of TVBackend +type PGBackend struct { + sslKey pgSSLKeyTable + dnssec pgDNSSecTable + uri pgURISignKeyTable + url pgURLSigKeyTable + cfg PGConfig + db *sql.DB +} + +// String returns a high level overview of the backend and its keys +func (pg *PGBackend) String() string { + data := fmt.Sprintf("PG server %s@%s:%s\n", pg.cfg.User, pg.cfg.Host, pg.cfg.Port) + data += fmt.Sprintf("\tSSL Keys: %d\n", len(pg.sslKey.Records)) + data += fmt.Sprintf("\tDNSSec Keys: %d\n", len(pg.dnssec.Records)) + data += fmt.Sprintf("\tURI Keys: %d\n", len(pg.uri.Records)) + data += fmt.Sprintf("\tURL Keys: %d\n", len(pg.url.Records)) + return data +} + +// Name returns the name for this backend +func (pg *PGBackend) Name() string { + return "PG" +} + +// ReadConfigFile takes in a filename and will read it into the backends config +func (pg *PGBackend) ReadConfigFile(configFile string) error { + var err error + if err = UnmarshalConfig(configFile, &pg.cfg); err != nil { + return err + } + + if pg.cfg.AESKey, err = base64.StdEncoding.DecodeString(pg.cfg.Key); err != nil { + return fmt.Errorf("unable to decode PG AESKey: %w", err) + } + return nil +} + +// Insert takes the current keys and inserts them into the backend DB Review comment: nit: GoDoc missing punctuation. ########## File path: tools/traffic_vault_migrate/postgres.go ########## @@ -0,0 +1,742 @@ +package main + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + util "github.com/apache/trafficcontrol/lib/go-util" + + _ "github.com/lib/pq" +) + +// PGConfig represents the configuration options available to the PG backend +type PGConfig struct { + Host string `json:"host"` + Port string `json:"port"` + User string `json:"user"` + Password string `json:"password"` + SSLMode string `json:"sslmode"` + Database string `json:"database"` + Key string `json:"aesKey"` + AESKey []byte +} + +// PGBackend is the Postgres implementation of TVBackend +type PGBackend struct { + sslKey pgSSLKeyTable + dnssec pgDNSSecTable + uri pgURISignKeyTable + url pgURLSigKeyTable + cfg PGConfig + db *sql.DB +} + +// String returns a high level overview of the backend and its keys +func (pg *PGBackend) String() string { + data := fmt.Sprintf("PG server %s@%s:%s\n", pg.cfg.User, pg.cfg.Host, pg.cfg.Port) + data += fmt.Sprintf("\tSSL Keys: %d\n", len(pg.sslKey.Records)) + data += fmt.Sprintf("\tDNSSec Keys: %d\n", len(pg.dnssec.Records)) + data += fmt.Sprintf("\tURI Keys: %d\n", len(pg.uri.Records)) + data += fmt.Sprintf("\tURL Keys: %d\n", len(pg.url.Records)) + return data +} + +// Name returns the name for this backend +func (pg *PGBackend) Name() string { + return "PG" +} + +// ReadConfigFile takes in a filename and will read it into the backends config +func (pg *PGBackend) ReadConfigFile(configFile string) error { + var err error + if err = UnmarshalConfig(configFile, &pg.cfg); err != nil { + return err + } + + if pg.cfg.AESKey, err = base64.StdEncoding.DecodeString(pg.cfg.Key); err != nil { + return fmt.Errorf("unable to decode PG AESKey: %w", err) + } + return nil +} + +// Insert takes the current keys and inserts them into the backend DB +func (pg *PGBackend) Insert() error { + if err := pg.sslKey.insertKeys(pg.db); err != nil { + return err + } + if err := pg.dnssec.insertKeys(pg.db); err != nil { + return err + } + if err := pg.url.insertKeys(pg.db); err != nil { + return err + } + if err := pg.uri.insertKeys(pg.db); err != nil { + return err + } + return nil +} + +// Start initiates the connection to the backend DB Review comment: nit: GoDoc missing punctuation. ########## File path: lib/go-tc/deliveryservice_ssl_keys.go ########## @@ -73,6 +73,7 @@ type SSLKeyRequestFields struct { HostName *string `json:"hostname,omitempty"` Country *string `json:"country,omitempty"` State *string `json:"state,omitempty"` + Version *int `json:"version,omitempty"` Review comment: I'm not sure we can add this without making a new-version structure, because this is used in the APIv3 client method `GenerateSSLKeysForDS` which has been published. That's what @rob05c would say, I think. @rawlinp will disagree, saying that it won't break existing clients because they couldn't have been using it anyway - which is true. Personally I don't care either way, but would find it easier to just make an `SSLKeyRequestV40` and `SSLKeyRequestV4` alias as it's a small enough change and won't hurt anything even if it's unnecessary. But if rob doesn't care I don't think anyone else will, so in that case don't worry about it. ########## File path: tools/traffic_vault_migrate/postgres.go ########## @@ -0,0 +1,742 @@ +package main + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + util "github.com/apache/trafficcontrol/lib/go-util" + + _ "github.com/lib/pq" +) + +// PGConfig represents the configuration options available to the PG backend +type PGConfig struct { + Host string `json:"host"` + Port string `json:"port"` + User string `json:"user"` + Password string `json:"password"` + SSLMode string `json:"sslmode"` + Database string `json:"database"` + Key string `json:"aesKey"` + AESKey []byte +} + +// PGBackend is the Postgres implementation of TVBackend +type PGBackend struct { + sslKey pgSSLKeyTable + dnssec pgDNSSecTable + uri pgURISignKeyTable + url pgURLSigKeyTable + cfg PGConfig + db *sql.DB +} + +// String returns a high level overview of the backend and its keys Review comment: nit: GoDoc missing punctuation. ########## File path: tools/traffic_vault_migrate/postgres.go ########## @@ -0,0 +1,742 @@ +package main + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + util "github.com/apache/trafficcontrol/lib/go-util" + + _ "github.com/lib/pq" +) + +// PGConfig represents the configuration options available to the PG backend +type PGConfig struct { + Host string `json:"host"` + Port string `json:"port"` + User string `json:"user"` + Password string `json:"password"` + SSLMode string `json:"sslmode"` + Database string `json:"database"` + Key string `json:"aesKey"` + AESKey []byte +} + +// PGBackend is the Postgres implementation of TVBackend +type PGBackend struct { + sslKey pgSSLKeyTable + dnssec pgDNSSecTable + uri pgURISignKeyTable + url pgURLSigKeyTable + cfg PGConfig + db *sql.DB +} + +// String returns a high level overview of the backend and its keys +func (pg *PGBackend) String() string { + data := fmt.Sprintf("PG server %s@%s:%s\n", pg.cfg.User, pg.cfg.Host, pg.cfg.Port) + data += fmt.Sprintf("\tSSL Keys: %d\n", len(pg.sslKey.Records)) + data += fmt.Sprintf("\tDNSSec Keys: %d\n", len(pg.dnssec.Records)) + data += fmt.Sprintf("\tURI Keys: %d\n", len(pg.uri.Records)) + data += fmt.Sprintf("\tURL Keys: %d\n", len(pg.url.Records)) + return data +} + +// Name returns the name for this backend +func (pg *PGBackend) Name() string { + return "PG" +} + +// ReadConfigFile takes in a filename and will read it into the backends config Review comment: nit: GoDoc missing punctuation. ########## File path: tools/traffic_vault_migrate/postgres.go ########## @@ -0,0 +1,742 @@ +package main + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + util "github.com/apache/trafficcontrol/lib/go-util" + + _ "github.com/lib/pq" +) + +// PGConfig represents the configuration options available to the PG backend +type PGConfig struct { + Host string `json:"host"` + Port string `json:"port"` + User string `json:"user"` + Password string `json:"password"` + SSLMode string `json:"sslmode"` + Database string `json:"database"` + Key string `json:"aesKey"` + AESKey []byte +} + +// PGBackend is the Postgres implementation of TVBackend +type PGBackend struct { + sslKey pgSSLKeyTable + dnssec pgDNSSecTable + uri pgURISignKeyTable + url pgURLSigKeyTable + cfg PGConfig + db *sql.DB +} + +// String returns a high level overview of the backend and its keys +func (pg *PGBackend) String() string { + data := fmt.Sprintf("PG server %s@%s:%s\n", pg.cfg.User, pg.cfg.Host, pg.cfg.Port) + data += fmt.Sprintf("\tSSL Keys: %d\n", len(pg.sslKey.Records)) + data += fmt.Sprintf("\tDNSSec Keys: %d\n", len(pg.dnssec.Records)) + data += fmt.Sprintf("\tURI Keys: %d\n", len(pg.uri.Records)) + data += fmt.Sprintf("\tURL Keys: %d\n", len(pg.url.Records)) + return data +} + +// Name returns the name for this backend +func (pg *PGBackend) Name() string { + return "PG" +} + +// ReadConfigFile takes in a filename and will read it into the backends config +func (pg *PGBackend) ReadConfigFile(configFile string) error { + var err error + if err = UnmarshalConfig(configFile, &pg.cfg); err != nil { + return err + } + + if pg.cfg.AESKey, err = base64.StdEncoding.DecodeString(pg.cfg.Key); err != nil { + return fmt.Errorf("unable to decode PG AESKey: %w", err) + } + return nil +} + +// Insert takes the current keys and inserts them into the backend DB +func (pg *PGBackend) Insert() error { + if err := pg.sslKey.insertKeys(pg.db); err != nil { + return err + } + if err := pg.dnssec.insertKeys(pg.db); err != nil { + return err + } + if err := pg.url.insertKeys(pg.db); err != nil { + return err + } + if err := pg.uri.insertKeys(pg.db); err != nil { + return err + } + return nil +} + +// Start initiates the connection to the backend DB +func (pg *PGBackend) Start() error { + sqlStr := fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=%s", pg.cfg.User, pg.cfg.Password, pg.cfg.Host, pg.cfg.Port, pg.cfg.Database, pg.cfg.SSLMode) + db, err := sql.Open("postgres", sqlStr) + if err != nil { + return fmt.Errorf("unable to start PG client: %w", err) + } + + pg.db = db + pg.sslKey = pgSSLKeyTable{} + pg.dnssec = pgDNSSecTable{} + pg.url = pgURLSigKeyTable{} + pg.uri = pgURISignKeyTable{} + + return nil +} + +// ValidateKey validates that the keys are valid (in most cases, certain fields are not null) +func (pg *PGBackend) ValidateKey() []string { + var errs []string + if errs := pg.sslKey.validate(); errs != nil { + errs = append(errs, errs...) + } + if errs := pg.dnssec.validate(); errs != nil { + errs = append(errs, errs...) + } + if errs := pg.uri.validate(); errs != nil { + errs = append(errs, errs...) + } + if errs := pg.url.validate(); errs != nil { + errs = append(errs, errs...) + } + return errs +} + +// Close terminates the connection to the backend DB +func (pg *PGBackend) Close() error { + return pg.db.Close() +} + +// Ping checks the connection to the backend DB +func (pg *PGBackend) Ping() error { + return pg.db.Ping() +} + +// Fetch gets all of the keys from the backend DB Review comment: nit: GoDoc missing punctuation. ########## File path: tools/traffic_vault_migrate/postgres.go ########## @@ -0,0 +1,742 @@ +package main + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + util "github.com/apache/trafficcontrol/lib/go-util" + + _ "github.com/lib/pq" +) + +// PGConfig represents the configuration options available to the PG backend Review comment: nit: GoDoc missing punctuation. ########## File path: tools/traffic_vault_migrate/postgres.go ########## @@ -0,0 +1,742 @@ +package main + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + util "github.com/apache/trafficcontrol/lib/go-util" + + _ "github.com/lib/pq" +) + +// PGConfig represents the configuration options available to the PG backend +type PGConfig struct { + Host string `json:"host"` + Port string `json:"port"` + User string `json:"user"` + Password string `json:"password"` + SSLMode string `json:"sslmode"` + Database string `json:"database"` + Key string `json:"aesKey"` + AESKey []byte +} + +// PGBackend is the Postgres implementation of TVBackend Review comment: nit: GoDoc missing punctuation. ########## File path: tools/traffic_vault_migrate/postgres.go ########## @@ -0,0 +1,742 @@ +package main + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + util "github.com/apache/trafficcontrol/lib/go-util" + + _ "github.com/lib/pq" +) + +// PGConfig represents the configuration options available to the PG backend +type PGConfig struct { + Host string `json:"host"` + Port string `json:"port"` + User string `json:"user"` + Password string `json:"password"` + SSLMode string `json:"sslmode"` + Database string `json:"database"` + Key string `json:"aesKey"` + AESKey []byte +} + +// PGBackend is the Postgres implementation of TVBackend +type PGBackend struct { + sslKey pgSSLKeyTable + dnssec pgDNSSecTable + uri pgURISignKeyTable + url pgURLSigKeyTable + cfg PGConfig + db *sql.DB +} + +// String returns a high level overview of the backend and its keys +func (pg *PGBackend) String() string { + data := fmt.Sprintf("PG server %s@%s:%s\n", pg.cfg.User, pg.cfg.Host, pg.cfg.Port) + data += fmt.Sprintf("\tSSL Keys: %d\n", len(pg.sslKey.Records)) + data += fmt.Sprintf("\tDNSSec Keys: %d\n", len(pg.dnssec.Records)) + data += fmt.Sprintf("\tURI Keys: %d\n", len(pg.uri.Records)) + data += fmt.Sprintf("\tURL Keys: %d\n", len(pg.url.Records)) + return data +} + +// Name returns the name for this backend +func (pg *PGBackend) Name() string { + return "PG" +} + +// ReadConfigFile takes in a filename and will read it into the backends config +func (pg *PGBackend) ReadConfigFile(configFile string) error { + var err error + if err = UnmarshalConfig(configFile, &pg.cfg); err != nil { + return err + } + + if pg.cfg.AESKey, err = base64.StdEncoding.DecodeString(pg.cfg.Key); err != nil { + return fmt.Errorf("unable to decode PG AESKey: %w", err) + } + return nil +} + +// Insert takes the current keys and inserts them into the backend DB +func (pg *PGBackend) Insert() error { + if err := pg.sslKey.insertKeys(pg.db); err != nil { + return err + } + if err := pg.dnssec.insertKeys(pg.db); err != nil { + return err + } + if err := pg.url.insertKeys(pg.db); err != nil { + return err + } + if err := pg.uri.insertKeys(pg.db); err != nil { + return err + } + return nil +} + +// Start initiates the connection to the backend DB +func (pg *PGBackend) Start() error { + sqlStr := fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=%s", pg.cfg.User, pg.cfg.Password, pg.cfg.Host, pg.cfg.Port, pg.cfg.Database, pg.cfg.SSLMode) + db, err := sql.Open("postgres", sqlStr) + if err != nil { + return fmt.Errorf("unable to start PG client: %w", err) + } + + pg.db = db + pg.sslKey = pgSSLKeyTable{} + pg.dnssec = pgDNSSecTable{} + pg.url = pgURLSigKeyTable{} + pg.uri = pgURISignKeyTable{} + + return nil +} + +// ValidateKey validates that the keys are valid (in most cases, certain fields are not null) Review comment: nit: GoDoc missing punctuation. ########## File path: tools/traffic_vault_migrate/postgres.go ########## @@ -0,0 +1,742 @@ +package main + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + util "github.com/apache/trafficcontrol/lib/go-util" + + _ "github.com/lib/pq" +) + +// PGConfig represents the configuration options available to the PG backend +type PGConfig struct { + Host string `json:"host"` + Port string `json:"port"` + User string `json:"user"` + Password string `json:"password"` + SSLMode string `json:"sslmode"` + Database string `json:"database"` + Key string `json:"aesKey"` + AESKey []byte +} + +// PGBackend is the Postgres implementation of TVBackend +type PGBackend struct { + sslKey pgSSLKeyTable + dnssec pgDNSSecTable + uri pgURISignKeyTable + url pgURLSigKeyTable + cfg PGConfig + db *sql.DB +} + +// String returns a high level overview of the backend and its keys +func (pg *PGBackend) String() string { + data := fmt.Sprintf("PG server %s@%s:%s\n", pg.cfg.User, pg.cfg.Host, pg.cfg.Port) + data += fmt.Sprintf("\tSSL Keys: %d\n", len(pg.sslKey.Records)) + data += fmt.Sprintf("\tDNSSec Keys: %d\n", len(pg.dnssec.Records)) + data += fmt.Sprintf("\tURI Keys: %d\n", len(pg.uri.Records)) + data += fmt.Sprintf("\tURL Keys: %d\n", len(pg.url.Records)) + return data +} + +// Name returns the name for this backend Review comment: nit: GoDoc missing punctuation. ########## File path: tools/traffic_vault_migrate/postgres.go ########## @@ -0,0 +1,742 @@ +package main + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + util "github.com/apache/trafficcontrol/lib/go-util" + + _ "github.com/lib/pq" +) + +// PGConfig represents the configuration options available to the PG backend +type PGConfig struct { + Host string `json:"host"` + Port string `json:"port"` + User string `json:"user"` + Password string `json:"password"` + SSLMode string `json:"sslmode"` + Database string `json:"database"` + Key string `json:"aesKey"` + AESKey []byte +} + +// PGBackend is the Postgres implementation of TVBackend +type PGBackend struct { + sslKey pgSSLKeyTable + dnssec pgDNSSecTable + uri pgURISignKeyTable + url pgURLSigKeyTable + cfg PGConfig + db *sql.DB +} + +// String returns a high level overview of the backend and its keys +func (pg *PGBackend) String() string { + data := fmt.Sprintf("PG server %s@%s:%s\n", pg.cfg.User, pg.cfg.Host, pg.cfg.Port) + data += fmt.Sprintf("\tSSL Keys: %d\n", len(pg.sslKey.Records)) + data += fmt.Sprintf("\tDNSSec Keys: %d\n", len(pg.dnssec.Records)) + data += fmt.Sprintf("\tURI Keys: %d\n", len(pg.uri.Records)) + data += fmt.Sprintf("\tURL Keys: %d\n", len(pg.url.Records)) + return data +} + +// Name returns the name for this backend +func (pg *PGBackend) Name() string { + return "PG" +} + +// ReadConfigFile takes in a filename and will read it into the backends config +func (pg *PGBackend) ReadConfigFile(configFile string) error { + var err error + if err = UnmarshalConfig(configFile, &pg.cfg); err != nil { + return err + } + + if pg.cfg.AESKey, err = base64.StdEncoding.DecodeString(pg.cfg.Key); err != nil { + return fmt.Errorf("unable to decode PG AESKey: %w", err) + } + return nil +} + +// Insert takes the current keys and inserts them into the backend DB +func (pg *PGBackend) Insert() error { + if err := pg.sslKey.insertKeys(pg.db); err != nil { + return err + } + if err := pg.dnssec.insertKeys(pg.db); err != nil { + return err + } + if err := pg.url.insertKeys(pg.db); err != nil { + return err + } + if err := pg.uri.insertKeys(pg.db); err != nil { + return err + } + return nil +} + +// Start initiates the connection to the backend DB +func (pg *PGBackend) Start() error { + sqlStr := fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=%s", pg.cfg.User, pg.cfg.Password, pg.cfg.Host, pg.cfg.Port, pg.cfg.Database, pg.cfg.SSLMode) + db, err := sql.Open("postgres", sqlStr) + if err != nil { + return fmt.Errorf("unable to start PG client: %w", err) + } + + pg.db = db + pg.sslKey = pgSSLKeyTable{} + pg.dnssec = pgDNSSecTable{} + pg.url = pgURLSigKeyTable{} + pg.uri = pgURISignKeyTable{} + + return nil +} + +// ValidateKey validates that the keys are valid (in most cases, certain fields are not null) +func (pg *PGBackend) ValidateKey() []string { + var errs []string + if errs := pg.sslKey.validate(); errs != nil { + errs = append(errs, errs...) + } + if errs := pg.dnssec.validate(); errs != nil { + errs = append(errs, errs...) + } + if errs := pg.uri.validate(); errs != nil { + errs = append(errs, errs...) + } + if errs := pg.url.validate(); errs != nil { + errs = append(errs, errs...) + } + return errs +} + +// Close terminates the connection to the backend DB Review comment: nit: GoDoc missing punctuation. ########## File path: tools/traffic_vault_migrate/postgres.go ########## @@ -0,0 +1,742 @@ +package main + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + util "github.com/apache/trafficcontrol/lib/go-util" + + _ "github.com/lib/pq" +) + +// PGConfig represents the configuration options available to the PG backend +type PGConfig struct { + Host string `json:"host"` + Port string `json:"port"` + User string `json:"user"` + Password string `json:"password"` + SSLMode string `json:"sslmode"` + Database string `json:"database"` + Key string `json:"aesKey"` + AESKey []byte +} + +// PGBackend is the Postgres implementation of TVBackend +type PGBackend struct { + sslKey pgSSLKeyTable + dnssec pgDNSSecTable + uri pgURISignKeyTable + url pgURLSigKeyTable + cfg PGConfig + db *sql.DB +} + +// String returns a high level overview of the backend and its keys +func (pg *PGBackend) String() string { + data := fmt.Sprintf("PG server %s@%s:%s\n", pg.cfg.User, pg.cfg.Host, pg.cfg.Port) + data += fmt.Sprintf("\tSSL Keys: %d\n", len(pg.sslKey.Records)) + data += fmt.Sprintf("\tDNSSec Keys: %d\n", len(pg.dnssec.Records)) + data += fmt.Sprintf("\tURI Keys: %d\n", len(pg.uri.Records)) + data += fmt.Sprintf("\tURL Keys: %d\n", len(pg.url.Records)) + return data +} + +// Name returns the name for this backend +func (pg *PGBackend) Name() string { + return "PG" +} + +// ReadConfigFile takes in a filename and will read it into the backends config +func (pg *PGBackend) ReadConfigFile(configFile string) error { + var err error + if err = UnmarshalConfig(configFile, &pg.cfg); err != nil { + return err + } + + if pg.cfg.AESKey, err = base64.StdEncoding.DecodeString(pg.cfg.Key); err != nil { + return fmt.Errorf("unable to decode PG AESKey: %w", err) + } + return nil +} + +// Insert takes the current keys and inserts them into the backend DB +func (pg *PGBackend) Insert() error { + if err := pg.sslKey.insertKeys(pg.db); err != nil { + return err + } + if err := pg.dnssec.insertKeys(pg.db); err != nil { + return err + } + if err := pg.url.insertKeys(pg.db); err != nil { + return err + } + if err := pg.uri.insertKeys(pg.db); err != nil { + return err + } + return nil +} + +// Start initiates the connection to the backend DB +func (pg *PGBackend) Start() error { + sqlStr := fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=%s", pg.cfg.User, pg.cfg.Password, pg.cfg.Host, pg.cfg.Port, pg.cfg.Database, pg.cfg.SSLMode) + db, err := sql.Open("postgres", sqlStr) + if err != nil { + return fmt.Errorf("unable to start PG client: %w", err) + } + + pg.db = db + pg.sslKey = pgSSLKeyTable{} + pg.dnssec = pgDNSSecTable{} + pg.url = pgURLSigKeyTable{} + pg.uri = pgURISignKeyTable{} + + return nil +} + +// ValidateKey validates that the keys are valid (in most cases, certain fields are not null) +func (pg *PGBackend) ValidateKey() []string { + var errs []string + if errs := pg.sslKey.validate(); errs != nil { + errs = append(errs, errs...) + } + if errs := pg.dnssec.validate(); errs != nil { + errs = append(errs, errs...) + } + if errs := pg.uri.validate(); errs != nil { + errs = append(errs, errs...) + } + if errs := pg.url.validate(); errs != nil { + errs = append(errs, errs...) + } + return errs +} + +// Close terminates the connection to the backend DB +func (pg *PGBackend) Close() error { + return pg.db.Close() +} + +// Ping checks the connection to the backend DB Review comment: nit: GoDoc missing punctuation. ########## File path: tools/traffic_vault_migrate/postgres.go ########## @@ -0,0 +1,742 @@ +package main + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + util "github.com/apache/trafficcontrol/lib/go-util" + + _ "github.com/lib/pq" +) + +// PGConfig represents the configuration options available to the PG backend +type PGConfig struct { + Host string `json:"host"` + Port string `json:"port"` + User string `json:"user"` + Password string `json:"password"` + SSLMode string `json:"sslmode"` + Database string `json:"database"` + Key string `json:"aesKey"` + AESKey []byte +} + +// PGBackend is the Postgres implementation of TVBackend +type PGBackend struct { + sslKey pgSSLKeyTable + dnssec pgDNSSecTable + uri pgURISignKeyTable + url pgURLSigKeyTable + cfg PGConfig + db *sql.DB +} + +// String returns a high level overview of the backend and its keys +func (pg *PGBackend) String() string { + data := fmt.Sprintf("PG server %s@%s:%s\n", pg.cfg.User, pg.cfg.Host, pg.cfg.Port) + data += fmt.Sprintf("\tSSL Keys: %d\n", len(pg.sslKey.Records)) + data += fmt.Sprintf("\tDNSSec Keys: %d\n", len(pg.dnssec.Records)) + data += fmt.Sprintf("\tURI Keys: %d\n", len(pg.uri.Records)) + data += fmt.Sprintf("\tURL Keys: %d\n", len(pg.url.Records)) + return data +} + +// Name returns the name for this backend +func (pg *PGBackend) Name() string { + return "PG" +} + +// ReadConfigFile takes in a filename and will read it into the backends config +func (pg *PGBackend) ReadConfigFile(configFile string) error { + var err error + if err = UnmarshalConfig(configFile, &pg.cfg); err != nil { + return err + } + + if pg.cfg.AESKey, err = base64.StdEncoding.DecodeString(pg.cfg.Key); err != nil { + return fmt.Errorf("unable to decode PG AESKey: %w", err) + } + return nil +} + +// Insert takes the current keys and inserts them into the backend DB +func (pg *PGBackend) Insert() error { + if err := pg.sslKey.insertKeys(pg.db); err != nil { + return err + } + if err := pg.dnssec.insertKeys(pg.db); err != nil { + return err + } + if err := pg.url.insertKeys(pg.db); err != nil { + return err + } + if err := pg.uri.insertKeys(pg.db); err != nil { + return err + } + return nil +} + +// Start initiates the connection to the backend DB +func (pg *PGBackend) Start() error { + sqlStr := fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=%s", pg.cfg.User, pg.cfg.Password, pg.cfg.Host, pg.cfg.Port, pg.cfg.Database, pg.cfg.SSLMode) + db, err := sql.Open("postgres", sqlStr) + if err != nil { + return fmt.Errorf("unable to start PG client: %w", err) + } + + pg.db = db + pg.sslKey = pgSSLKeyTable{} + pg.dnssec = pgDNSSecTable{} + pg.url = pgURLSigKeyTable{} + pg.uri = pgURISignKeyTable{} + + return nil +} + +// ValidateKey validates that the keys are valid (in most cases, certain fields are not null) +func (pg *PGBackend) ValidateKey() []string { + var errs []string + if errs := pg.sslKey.validate(); errs != nil { + errs = append(errs, errs...) + } + if errs := pg.dnssec.validate(); errs != nil { + errs = append(errs, errs...) + } + if errs := pg.uri.validate(); errs != nil { + errs = append(errs, errs...) + } + if errs := pg.url.validate(); errs != nil { + errs = append(errs, errs...) + } + return errs +} + +// Close terminates the connection to the backend DB +func (pg *PGBackend) Close() error { + return pg.db.Close() +} + +// Ping checks the connection to the backend DB +func (pg *PGBackend) Ping() error { + return pg.db.Ping() +} + +// Fetch gets all of the keys from the backend DB +func (pg *PGBackend) Fetch() error { + if err := pg.sslKey.gatherKeys(pg.db); err != nil { + return err + } + + if err := pg.dnssec.gatherKeys(pg.db); err != nil { + return err + } + + if err := pg.url.gatherKeys(pg.db); err != nil { + return err + } + + if err := pg.uri.gatherKeys(pg.db); err != nil { + return err + } + + return nil +} + +// GetSSLKeys converts the backends internal key representation into the common representation (SSLKey) +func (pg *PGBackend) GetSSLKeys() ([]SSLKey, error) { + if err := pg.sslKey.decrypt(pg.cfg.AESKey); err != nil { + return nil, err + } + return pg.sslKey.toGeneric(), nil +} + +// SetSSLKeys takes in keys and converts & encrypts the data into the backends internal format +func (pg *PGBackend) SetSSLKeys(keys []SSLKey) error { + pg.sslKey.fromGeneric(keys) + return pg.sslKey.encrypt(pg.cfg.AESKey) +} + +// GetDNSSecKeys converts the backends internal key representation into the common representation (DNSSecKey) +func (pg *PGBackend) GetDNSSecKeys() ([]DNSSecKey, error) { + if err := pg.dnssec.decrypt(pg.cfg.AESKey); err != nil { + return nil, err + } + return pg.dnssec.toGeneric(), nil +} + +// SetDNSSecKeys takes in keys and converts & encrypts the data into the backends internal format +func (pg *PGBackend) SetDNSSecKeys(keys []DNSSecKey) error { + pg.dnssec.fromGeneric(keys) + return pg.dnssec.encrypt(pg.cfg.AESKey) +} + +// GetURISignKeys converts the pg internal key representation into the common representation (URISignKey) +func (pg *PGBackend) GetURISignKeys() ([]URISignKey, error) { + if err := pg.uri.decrypt(pg.cfg.AESKey); err != nil { + return nil, err + } + return pg.uri.toGeneric(), nil +} + +// SetURISignKeys takes in keys and converts & encrypts the data into the backends internal format +func (pg *PGBackend) SetURISignKeys(keys []URISignKey) error { + pg.uri.fromGeneric(keys) + return pg.uri.encrypt(pg.cfg.AESKey) +} + +// GetURLSigKeys converts the backends internal key representation into the common representation (URLSigKey) +func (pg *PGBackend) GetURLSigKeys() ([]URLSigKey, error) { + if err := pg.url.decrypt(pg.cfg.AESKey); err != nil { + return nil, err + } + return pg.url.toGeneric(), nil +} + +// SetURLSigKeys takes in keys and converts & encrypts the data into the backends internal format Review comment: nit: GoDoc missing punctuation. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
