This is an automated email from the ASF dual-hosted git repository.
mitchell852 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new fab1f00 Rewrote /dbdump to Go (#3912)
fab1f00 is described below
commit fab1f00ef55ac9381964ee7d4294f5d34fa731ff
Author: ocket8888 <[email protected]>
AuthorDate: Tue Sep 24 08:27:23 2019 -0600
Rewrote /dbdump to Go (#3912)
* Rewrote /dbdump to Go
* updated CHANGELOG.md
* Added docs
* added a test
* Fixed trailing paren
Also now cleaning up files after diff
* Switched format back to binary
---
CHANGELOG.md | 1 +
docs/source/api/dbdump.rst | 66 ++++++++++++++
.../cdn-in-a-box/traffic_ops/Dockerfile-go | 5 +-
.../traffic_ops_integration_test/Dockerfile | 2 +-
.../traffic_ops_integration_test/run.sh | 8 +-
traffic_ops/traffic_ops_golang/dbdump/dbdump.go | 100 +++++++++++++++++++++
traffic_ops/traffic_ops_golang/routing/routes.go | 4 +
7 files changed, 182 insertions(+), 4 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a083263..b0bcff7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -21,6 +21,7 @@ The format is based on [Keep a
Changelog](http://keepachangelog.com/en/1.0.0/).
- /api/1.4/cdns/name/:name/dnsseckeys `GET`
- /api/1.4/user/login/oauth `POST`
- /api/1.1/profiles/:name/configfiles/ats/* `GET`
+ - /api/1.1/dbdump `GET`
- To support reusing a single riak cluster connection, an optional parameter
is added to riak.conf: "HealthCheckInterval". This options takes a 'Duration'
value (ie: 10s, 5m) which affects how often the riak cluster is health checked.
Default is currently set to: "HealthCheckInterval": "5s".
- Added a new Go db/admin binary to replace the Perl db/admin.pl script which
is now deprecated and will be removed in a future release. The new db/admin
binary is essentially a drop-in replacement for db/admin.pl since it supports
all of the same commands and options; therefore, it should be used in place of
db/admin.pl for all the same tasks.
- Added an API 1.4 endpoint, /api/1.4/cdns/dnsseckeys/refresh, to perform
necessary behavior previously served outside the API under `/internal`.
diff --git a/docs/source/api/dbdump.rst b/docs/source/api/dbdump.rst
new file mode 100644
index 0000000..1880954
--- /dev/null
+++ b/docs/source/api/dbdump.rst
@@ -0,0 +1,66 @@
+..
+..
+.. Licensed under the Apache License, Version 2.0 (the "License");
+.. you may not use this file except in compliance with the License.
+.. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+..
+
+.. _to-api-dbdump:
+
+**********
+``dbdump``
+**********
+.. caution:: This is an extremely dangerous thing to do, as it exposes the
entirety of the database, including possibly sensitive information.
Administrators and systems engineers are advised to instead use
database-specific tools to make server transitions more securely.
+
+Dumps the Traffic Ops database as an SQL script that should recreate its
schema and contents exactly.
+
+.. impl-detail:: The script is output using the :manpage:`pg_dump(1)` utility,
and is thus compatible for use with the :manpage:`pg_restore(1)` utility.
+
+``GET``
+=======
+Fetches the database dump.
+
+:Auth. Required: Yes
+:Roles Required: "admin"
+:Response Type: ``undefined`` - outputs an SQL script, not JSON
+
+Request Structure
+-----------------
+No parameters available
+
+.. code-block:: http
+ :caption: Request Example
+
+ GET /api/1.4/dbdump HTTP/1.1
+ Host: trafficops.infra.ciab.test
+ User-Agent: curl/7.47.0
+ Accept: */*
+ Cookie: mojolicious=...
+
+Response Structure
+------------------
+.. code-block:: http
+ :caption: Response Example
+
+ HTTP/1.1 200 OK
+ Access-Control-Allow-Credentials: true
+ Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type,
Accept, Set-Cookie, Cookie
+ Access-Control-Allow-Methods: POST,GET,OPTIONS,PUT,DELETE
+ Access-Control-Allow-Origin: *
+ Content-Type: application/sql
+ Content-Disposition: attachment
+ Set-Cookie: mojolicious=...; Path=/; HttpOnly
+ Whole-Content-Sha512:
YwvPB0ZToyzT8ilBnDlWWdwV+E3f2Xgus1OKrkNaipQqgrw5zGwq0rC1U9TZ8Zl6kAGcRZgCYnr1EWfHXpJRkg==
+ X-Server-Name: traffic_ops_golang/
+ Date: Mon, 09 Sep 2019 21:08:28 GMT
+ Transfer-Encoding: chunked
+
+ -- Actual text omitted - it's huge
diff --git a/infrastructure/cdn-in-a-box/traffic_ops/Dockerfile-go
b/infrastructure/cdn-in-a-box/traffic_ops/Dockerfile-go
index 13d5070..cbb62e1 100644
--- a/infrastructure/cdn-in-a-box/traffic_ops/Dockerfile-go
+++ b/infrastructure/cdn-in-a-box/traffic_ops/Dockerfile-go
@@ -22,8 +22,9 @@
FROM centos:7
-RUN yum -y install epel-release && \
- yum -y install jq bind-utils net-tools gettext perl-JSON-PP nmap-ncat
openssl && \
+RUN rpm -Uvh
http://yum.postgresql.org/9.6/redhat/rhel-7-x86_64/pgdg-redhat96-9.6-3.noarch.rpm
&& \
+ yum -y install epel-release && \
+ yum -y install jq bind-utils net-tools gettext perl-JSON-PP nmap-ncat
openssl postgresql96 && \
yum clean all
EXPOSE 443
diff --git
a/infrastructure/cdn-in-a-box/traffic_ops_integration_test/Dockerfile
b/infrastructure/cdn-in-a-box/traffic_ops_integration_test/Dockerfile
index 496b5f0..880dbd3 100644
--- a/infrastructure/cdn-in-a-box/traffic_ops_integration_test/Dockerfile
+++ b/infrastructure/cdn-in-a-box/traffic_ops_integration_test/Dockerfile
@@ -36,7 +36,7 @@ RUN go get -u github.com/jstemmer/go-junit-report
RUN cd /go/src/github.com/jstemmer/go-junit-report && go build
FROM debian:stretch
-RUN apt-get update && apt-get install -y netcat curl dnsutils net-tools vim &&
apt-get clean
+RUN apt-get update && apt-get install -y netcat curl dnsutils net-tools
postgresql-client && apt-get clean
# MANIFEST
# run.sh (wait on TO, then run bin)
diff --git a/infrastructure/cdn-in-a-box/traffic_ops_integration_test/run.sh
b/infrastructure/cdn-in-a-box/traffic_ops_integration_test/run.sh
index fe7a443..5b44953 100755
--- a/infrastructure/cdn-in-a-box/traffic_ops_integration_test/run.sh
+++ b/infrastructure/cdn-in-a-box/traffic_ops_integration_test/run.sh
@@ -39,4 +39,10 @@ done
# if [[ -x ]]; then;./config.sh; done traffic_ops/run-go.sh
source config.sh
-./traffic_ops_integration_test -test.v -cfg=traffic-ops-test.conf 2>&1 |
./go-junit-report --package-name=golang.test.toapi --set-exit-code >
/junit/golang.test.toapi.xml && chmod 777 -R /junit && cat
/junit/golang.test.toapi.xml
+PGPASSWORD="$DB_USER_PASS" pg_dump --blobs --no-owner --format=c
"--host=$DB_SERVER" "--port=$DB_PORT" "--username=$DB_USER" traffic_ops >
dbdump.manual
+to-get api/1.4/dbdump > dbdump.api
+
+diff dbdump.api dbdump.manual && rm -f dbdump.api dbdump.manual
+
+./traffic_ops_integration_test -test.v -cfg=traffic-ops-test.conf 2>&1 |
./go-junit-report --package-name=golang.test.toapi --set-exit-code >
/junit/golang.test.toapi.xml && find /junit -type 'f' | xargs chmod 664 && cat
/junit/golang.test.toapi.xml
+sleep 8000
diff --git a/traffic_ops/traffic_ops_golang/dbdump/dbdump.go
b/traffic_ops/traffic_ops_golang/dbdump/dbdump.go
new file mode 100644
index 0000000..5912444
--- /dev/null
+++ b/traffic_ops/traffic_ops_golang/dbdump/dbdump.go
@@ -0,0 +1,100 @@
+package dbdump
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import "errors"
+import "fmt"
+import "net/http"
+import "os"
+import "os/exec"
+import "time"
+
+import "github.com/apache/trafficcontrol/lib/go-log"
+import "github.com/apache/trafficcontrol/lib/go-tc"
+
+import "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api"
+
+func filename() string {
+ host, err := os.Hostname()
+ if err != nil {
+ host = "UNKNOWN"
+ log.Warnf("Unable to determine hostname: %v", err)
+ }
+
+ return fmt.Sprintf("to-backup-%s-%s.pg_dump", host,
time.Now().Format(time.RFC3339))
+}
+
+func DBDump(w http.ResponseWriter, r *http.Request) {
+ inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil)
+ tx := inf.Tx.Tx
+ if userErr != nil || sysErr != nil {
+ api.HandleErr(w, r, tx, errCode, userErr, sysErr)
+ return
+ }
+ defer inf.Close()
+
+ conf := inf.Config.DB
+
+ pgdump, err := exec.LookPath("pg_dump")
+ if err != nil {
+ sysErr = fmt.Errorf("Looking up 'pg_dump' executable: %v", err)
+ userErr = errors.New("'pg_dump' not available")
+ errCode = http.StatusServiceUnavailable
+ api.HandleErr(w, r, tx, errCode, userErr, sysErr)
+ return
+ }
+
+ cmd := exec.Cmd{
+ Path: pgdump,
+ Args: []string{
+ "--blobs",
+ "--no-owner",
+ "--format=c",
+ fmt.Sprintf("--host=%s", conf.Hostname),
+ fmt.Sprintf("--port=%s", conf.Port),
+ fmt.Sprintf("--username=%s", conf.User),
+ conf.DBName,
+ },
+ Env: []string{
+ fmt.Sprintf("PGPASSWORD=%s", conf.Password),
+ },
+ }
+
+ out, err := cmd.Output()
+ if err != nil {
+ switch err.(type) {
+ case *exec.ExitError:
+ sysErr = fmt.Errorf("subprocess encountered an error,
stderr: %s", err.(*exec.ExitError).Stderr)
+ default:
+ sysErr = fmt.Errorf("subprocess encountered an error:
%v", err)
+ }
+ userErr = errors.New("Subprocess encountered an error")
+ errCode = http.StatusBadGateway
+ api.HandleErr(w, r, tx, errCode, userErr, sysErr)
+ return
+ }
+
+ w.Header().Set(tc.ContentType,
"application/octet-stream;type=pg_dump-data")
+ w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment;
filename="%s"`, filename()))
+ if out[len(out)-1] != '\n' {
+ out = append(out, '\n')
+ }
+ w.Write(out)
+}
diff --git a/traffic_ops/traffic_ops_golang/routing/routes.go
b/traffic_ops/traffic_ops_golang/routing/routes.go
index 8e779ef..62685f7 100644
--- a/traffic_ops/traffic_ops_golang/routing/routes.go
+++ b/traffic_ops/traffic_ops_golang/routing/routes.go
@@ -46,6 +46,7 @@ import (
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cdnfederation"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/coordinate"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/crconfig"
+ "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbdump"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservice"
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservice/consistenthash"
dsrequest
"github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservice/request"
@@ -156,6 +157,9 @@ func Routes(d ServerData) ([]Route, []RawRoute,
http.Handler, error) {
//CDN: Monitoring: Traffic Monitor
{1.1, http.MethodGet,
`cdns/{cdn}/configs/monitoring(\.json)?$`,
crconfig.SnapshotGetMonitoringHandler, auth.PrivLevelReadOnly, Authenticated,
nil},
+ //Database dumps
+ {1.1, http.MethodGet, `dbdump/?`, dbdump.DBDump,
auth.PrivLevelAdmin, Authenticated, nil},
+
//Division: CRUD
{1.1, http.MethodGet, `divisions/?(\.json)?$`,
api.ReadHandler(&division.TODivision{}), auth.PrivLevelReadOnly, Authenticated,
nil},
{1.1, http.MethodGet, `divisions/{id}$`,
api.ReadHandler(&division.TODivision{}), auth.PrivLevelReadOnly, Authenticated,
nil},
