This is an automated email from the ASF dual-hosted git repository.
rawlin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new 9fa6c29 Skip check for the past dbversion's migration existing if it
is squashed (#6128)
9fa6c29 is described below
commit 9fa6c299271ef06075eec711ff1ff6805093f92b
Author: Zach Hoffman <[email protected]>
AuthorDate: Mon Aug 23 16:54:37 2021 -0600
Skip check for the past dbversion's migration existing if it is squashed
(#6128)
* Make ConnectionString a global
* Get the migration version after migrating from Goose
* Skip check for the past dbversion's migration existing if it is squashed
* Install TO dependencies before installing TO in db-admin dockerfile
* TO DB tests bugfix: Specify the database name with `dropdb`
* Only fail on a non-latest migration being modified/added if that
migration was changed in the PR or commit the TO DB tests GHA is running on
---
.github/actions/todb-tests/entrypoint.sh | 11 ++++-
traffic_ops/app/db/admin.go | 59 ++++++++++++++++++++------
traffic_ops_db/test/docker/Dockerfile-db-admin | 38 ++++++++++++-----
traffic_ops_db/test/docker/run-db-test.sh | 2 +-
4 files changed, 84 insertions(+), 26 deletions(-)
diff --git a/.github/actions/todb-tests/entrypoint.sh
b/.github/actions/todb-tests/entrypoint.sh
index 793c2c0..80702eb 100755
--- a/.github/actions/todb-tests/entrypoint.sh
+++ b/.github/actions/todb-tests/entrypoint.sh
@@ -18,6 +18,15 @@
set -e
+files_changed_in_pr() {
+ if [[ "$GITHUB_REF" == refs/pull/*/merge ]]; then
+ pr_number="$(<<<"$GITHUB_REF" grep -o '[0-9]\+')"
+ files_changed="$(curl
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${pr_number}/files" | jq -r
.[].filename)"
+ else
+ files_changed="$(git diff-tree --no-commit-id --name-only -r
"$GITHUB_SHA")"
+ fi
+}
+
CODE=0;
migration_dirs=(traffic_ops/app/db/migrations
traffic_ops/app/db/trafficvault/migrations);
@@ -69,7 +78,7 @@ for migration_dir in ${migration_dirs[@]}; do
done
mtime_length=${#mtime_array[@]}
- if [[ $LATEST_FILE_TIME != ${mtime_array[$mtime_length-1]} ]]; then
+ if [[ $LATEST_FILE_TIME != ${mtime_array[$mtime_length-1]} ]] &&
<<<"$(files_changed_in_pr)" grep -q "^${LATEST_FILE}$"; then
echo "ERROR: latest added/modified file: $LATEST_FILE is not in
the right order" >&2;
CODE=1;
fi
diff --git a/traffic_ops/app/db/admin.go b/traffic_ops/app/db/admin.go
index ce2a883..7f62287 100644
--- a/traffic_ops/app/db/admin.go
+++ b/traffic_ops/app/db/admin.go
@@ -34,7 +34,9 @@ import (
"github.com/apache/trafficcontrol/lib/go-log"
"github.com/golang-migrate/migrate/v4"
+ "github.com/golang-migrate/migrate/v4/database"
_ "github.com/golang-migrate/migrate/v4/database/postgres"
+ "github.com/golang-migrate/migrate/v4/source"
_ "github.com/golang-migrate/migrate/v4/source/file"
"gopkg.in/yaml.v2"
)
@@ -129,16 +131,17 @@ var (
DBVersionDirty bool
// globals that are parsed out of DBConfigFile and used in commands
- DBDriver string
- DBName string
- DBSuperUser = DefaultDBSuperUser
- DBUser string
- DBPassword string
- HostIP string
- HostPort string
- SSLMode string
- Migrate *migrate.Migrate
- MigrationName string
+ ConnectionString string
+ DBDriver string
+ DBName string
+ DBSuperUser = DefaultDBSuperUser
+ DBUser string
+ DBPassword string
+ HostIP string
+ HostPort string
+ SSLMode string
+ Migrate *migrate.Migrate
+ MigrationName string
)
func parseDBConfig() error {
@@ -330,13 +333,41 @@ func maybeMigrateFromGoose() bool {
if err := Migrate.Steps(1); err != nil {
die("Error migrating to Migrate from Goose: " + err.Error())
}
+ DBVersion, DBVersionDirty, _ = Migrate.Version()
return true
}
+// runFirstMigration is essentially Migrate.Migrate(FirstMigrationTimestamp)
but without the obligatory Migrate.versionExists() call.
+// If calling Migrate.versionExists() is made optional, runFirstMigration()
can be replaced.
+func runFirstMigration() error {
+ sourceDriver, sourceDriverErr := source.Open(DBMigrationsSource)
+ if sourceDriverErr != nil {
+ return fmt.Errorf("opening the migration source driver: " +
sourceDriverErr.Error())
+ }
+ dbDriver, dbDriverErr := database.Open(ConnectionString)
+ if dbDriverErr != nil {
+ return fmt.Errorf("opening the dbdriver: " +
dbDriverErr.Error())
+ }
+ firstMigration, firstMigrationName, migrationReadErr :=
sourceDriver.ReadUp(FirstMigrationTimestamp)
+ if migrationReadErr != nil {
+ return fmt.Errorf("reading migration %s: %s",
firstMigrationName, migrationReadErr.Error())
+ }
+ if setDirtyVersionErr :=
dbDriver.SetVersion(int(FirstMigrationTimestamp), true); setDirtyVersionErr !=
nil {
+ return fmt.Errorf("setting the dirty version: %s",
setDirtyVersionErr.Error())
+ }
+ if migrateErr := dbDriver.Run(firstMigration); migrateErr != nil {
+ return fmt.Errorf("running the migration: %s",
migrateErr.Error())
+ }
+ if setVersionErr := dbDriver.SetVersion(int(FirstMigrationTimestamp),
false); setVersionErr != nil {
+ return fmt.Errorf("setting the version after successfully
running the migration: %s", setVersionErr.Error())
+ }
+ return nil
+}
+
func runMigrations() {
migratedFromGoose := initMigrate()
if !TrafficVault && DBVersion == LastSquashedMigrationTimestamp &&
!DBVersionDirty {
- if migrateErr := Migrate.Migrate(FirstMigrationTimestamp);
migrateErr != nil {
+ if migrateErr := runFirstMigration(); migrateErr != nil {
die(fmt.Sprintf("Error migrating from DB version %d to
%d: %s", LastSquashedMigrationTimestamp, FirstMigrationTimestamp,
migrateErr.Error()))
}
}
@@ -569,11 +600,11 @@ func main() {
func initMigrate() bool {
var err error
- connectionString := fmt.Sprintf("%s://%s:%s@%s:%s/%s?sslmode=%s",
DBDriver, DBUser, DBPassword, HostIP, HostPort, DBName, SSLMode)
+ ConnectionString = fmt.Sprintf("%s://%s:%s@%s:%s/%s?sslmode=%s",
DBDriver, DBUser, DBPassword, HostIP, HostPort, DBName, SSLMode)
if TrafficVault {
- Migrate, err = migrate.New(TrafficVaultMigrationsSource,
connectionString)
+ Migrate, err = migrate.New(TrafficVaultMigrationsSource,
ConnectionString)
} else {
- Migrate, err = migrate.New(DBMigrationsSource, connectionString)
+ Migrate, err = migrate.New(DBMigrationsSource, ConnectionString)
}
if err != nil {
die("Starting Migrate: " + err.Error())
diff --git a/traffic_ops_db/test/docker/Dockerfile-db-admin
b/traffic_ops_db/test/docker/Dockerfile-db-admin
index 6f60481..03745c9 100644
--- a/traffic_ops_db/test/docker/Dockerfile-db-admin
+++ b/traffic_ops_db/test/docker/Dockerfile-db-admin
@@ -23,25 +23,43 @@ FROM centos:7
ARG POSTGRES_VERSION=13.2
ENV POSTGRES_VERSION=$POSTGRES_VERSION
-# NOTE: temporary workaround for removal of golang packages from CentOS 7 base
repo
RUN yum install -y \
- epel-release \
- centos-release-scl-rh \
-
https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm
\
+ epel-release \
+ centos-release-scl-rh \
+
https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm
\
git && \
- yum -y install golang
+ yum -y install \
+ cpanminus \
+ expat-devel \
+ libcap \
+ libcurl-devel \
+ libidn-devel \
+ libpcap-devel \
+ mkisofs \
+ openssl-devel \
+ perl-core \
+ perl-Crypt-ScryptKDF \
+ perl-DBD-Pg \
+ perl-DBI \
+ perl-Digest-SHA1 \
+ perl-JSON \
+ perl-libwww-perl \
+ perl-TermReadKey \
+ perl-Test-CPAN-Meta \
+ perl-WWW-Curl \
+ postgresql13 \
+ postgresql13-devel &&\
+ yum clean all
# Override TRAFFIC_OPS_RPM arg to use a different one using --build-arg
TRAFFIC_OPS_RPM=... Can be local file or http://...
ARG TRAFFIC_OPS_RPM=traffic_ops.rpm
ADD $TRAFFIC_OPS_RPM /
-RUN yum install -y \
- /$(basename $TRAFFIC_OPS_RPM) && \
- rm /$(basename $TRAFFIC_OPS_RPM) && \
- yum clean all
+RUN rpm -Uvh $(basename $TRAFFIC_OPS_RPM) && \
+ rm $(basename $TRAFFIC_OPS_RPM)
WORKDIR /opt/traffic_ops/app
-ADD run-db-test.sh \
+COPY run-db-test.sh \
db-config.sh \
/
diff --git a/traffic_ops_db/test/docker/run-db-test.sh
b/traffic_ops_db/test/docker/run-db-test.sh
index ff476d6..3bbcb59 100755
--- a/traffic_ops_db/test/docker/run-db-test.sh
+++ b/traffic_ops_db/test/docker/run-db-test.sh
@@ -132,7 +132,7 @@ fi
# test full restoration of the initial DB dump
for d in $(get_db_dumps); do
echo "testing restoration of DB dump: $d"
- dropdb --echo --if-exists < "$d" > /dev/null || echo "Dropping DB
${DB_NAME} failed: $d"
+ dropdb --echo --if-exists "$DB_NAME" < "$d" > /dev/null || echo "Dropping
DB ${DB_NAME} failed: $d"
createdb --echo < "$d" > /dev/null || echo "Creating DB ${DB_NAME} failed:
$d"
pg_restore --verbose --clean --if-exists --exit-on-error -d "$DB_NAME" <
"$d" > /dev/null || { echo "DB restoration failed: $d"; exit 1; }
done
