This is an automated email from the ASF dual-hosted git repository. zrhoffman pushed a commit to branch 6.0.x in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
commit 1e6a3c76a487a20e0e9d5226b03e612aa9071d8b Author: Zach Hoffman <[email protected]> AuthorDate: Mon Aug 23 16:54:37 2021 -0600 Skip check for the past dbversion's migration existing if it is squashed (#6128) * Make ConnectionString a global * Get the migration version after migrating from Goose * Skip check for the past dbversion's migration existing if it is squashed * Install TO dependencies before installing TO in db-admin dockerfile * TO DB tests bugfix: Specify the database name with `dropdb` * Only fail on a non-latest migration being modified/added if that migration was changed in the PR or commit the TO DB tests GHA is running on (cherry picked from commit 9fa6c299271ef06075eec711ff1ff6805093f92b) --- .github/actions/todb-tests/entrypoint.sh | 11 ++++- traffic_ops/app/db/admin.go | 59 ++++++++++++++++++++------ traffic_ops_db/test/docker/Dockerfile-db-admin | 38 ++++++++++++----- traffic_ops_db/test/docker/run-db-test.sh | 2 +- 4 files changed, 84 insertions(+), 26 deletions(-) diff --git a/.github/actions/todb-tests/entrypoint.sh b/.github/actions/todb-tests/entrypoint.sh index 793c2c0..80702eb 100755 --- a/.github/actions/todb-tests/entrypoint.sh +++ b/.github/actions/todb-tests/entrypoint.sh @@ -18,6 +18,15 @@ set -e +files_changed_in_pr() { + if [[ "$GITHUB_REF" == refs/pull/*/merge ]]; then + pr_number="$(<<<"$GITHUB_REF" grep -o '[0-9]\+')" + files_changed="$(curl "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/pulls/${pr_number}/files" | jq -r .[].filename)" + else + files_changed="$(git diff-tree --no-commit-id --name-only -r "$GITHUB_SHA")" + fi +} + CODE=0; migration_dirs=(traffic_ops/app/db/migrations traffic_ops/app/db/trafficvault/migrations); @@ -69,7 +78,7 @@ for migration_dir in ${migration_dirs[@]}; do done mtime_length=${#mtime_array[@]} - if [[ $LATEST_FILE_TIME != ${mtime_array[$mtime_length-1]} ]]; then + if [[ $LATEST_FILE_TIME != ${mtime_array[$mtime_length-1]} ]] && <<<"$(files_changed_in_pr)" grep -q "^${LATEST_FILE}$"; then echo "ERROR: latest added/modified file: $LATEST_FILE is not in the right order" >&2; CODE=1; fi diff --git a/traffic_ops/app/db/admin.go b/traffic_ops/app/db/admin.go index ce2a883..7f62287 100644 --- a/traffic_ops/app/db/admin.go +++ b/traffic_ops/app/db/admin.go @@ -34,7 +34,9 @@ import ( "github.com/apache/trafficcontrol/lib/go-log" "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" _ "github.com/golang-migrate/migrate/v4/database/postgres" + "github.com/golang-migrate/migrate/v4/source" _ "github.com/golang-migrate/migrate/v4/source/file" "gopkg.in/yaml.v2" ) @@ -129,16 +131,17 @@ var ( DBVersionDirty bool // globals that are parsed out of DBConfigFile and used in commands - DBDriver string - DBName string - DBSuperUser = DefaultDBSuperUser - DBUser string - DBPassword string - HostIP string - HostPort string - SSLMode string - Migrate *migrate.Migrate - MigrationName string + ConnectionString string + DBDriver string + DBName string + DBSuperUser = DefaultDBSuperUser + DBUser string + DBPassword string + HostIP string + HostPort string + SSLMode string + Migrate *migrate.Migrate + MigrationName string ) func parseDBConfig() error { @@ -330,13 +333,41 @@ func maybeMigrateFromGoose() bool { if err := Migrate.Steps(1); err != nil { die("Error migrating to Migrate from Goose: " + err.Error()) } + DBVersion, DBVersionDirty, _ = Migrate.Version() return true } +// runFirstMigration is essentially Migrate.Migrate(FirstMigrationTimestamp) but without the obligatory Migrate.versionExists() call. +// If calling Migrate.versionExists() is made optional, runFirstMigration() can be replaced. +func runFirstMigration() error { + sourceDriver, sourceDriverErr := source.Open(DBMigrationsSource) + if sourceDriverErr != nil { + return fmt.Errorf("opening the migration source driver: " + sourceDriverErr.Error()) + } + dbDriver, dbDriverErr := database.Open(ConnectionString) + if dbDriverErr != nil { + return fmt.Errorf("opening the dbdriver: " + dbDriverErr.Error()) + } + firstMigration, firstMigrationName, migrationReadErr := sourceDriver.ReadUp(FirstMigrationTimestamp) + if migrationReadErr != nil { + return fmt.Errorf("reading migration %s: %s", firstMigrationName, migrationReadErr.Error()) + } + if setDirtyVersionErr := dbDriver.SetVersion(int(FirstMigrationTimestamp), true); setDirtyVersionErr != nil { + return fmt.Errorf("setting the dirty version: %s", setDirtyVersionErr.Error()) + } + if migrateErr := dbDriver.Run(firstMigration); migrateErr != nil { + return fmt.Errorf("running the migration: %s", migrateErr.Error()) + } + if setVersionErr := dbDriver.SetVersion(int(FirstMigrationTimestamp), false); setVersionErr != nil { + return fmt.Errorf("setting the version after successfully running the migration: %s", setVersionErr.Error()) + } + return nil +} + func runMigrations() { migratedFromGoose := initMigrate() if !TrafficVault && DBVersion == LastSquashedMigrationTimestamp && !DBVersionDirty { - if migrateErr := Migrate.Migrate(FirstMigrationTimestamp); migrateErr != nil { + if migrateErr := runFirstMigration(); migrateErr != nil { die(fmt.Sprintf("Error migrating from DB version %d to %d: %s", LastSquashedMigrationTimestamp, FirstMigrationTimestamp, migrateErr.Error())) } } @@ -569,11 +600,11 @@ func main() { func initMigrate() bool { var err error - connectionString := fmt.Sprintf("%s://%s:%s@%s:%s/%s?sslmode=%s", DBDriver, DBUser, DBPassword, HostIP, HostPort, DBName, SSLMode) + ConnectionString = fmt.Sprintf("%s://%s:%s@%s:%s/%s?sslmode=%s", DBDriver, DBUser, DBPassword, HostIP, HostPort, DBName, SSLMode) if TrafficVault { - Migrate, err = migrate.New(TrafficVaultMigrationsSource, connectionString) + Migrate, err = migrate.New(TrafficVaultMigrationsSource, ConnectionString) } else { - Migrate, err = migrate.New(DBMigrationsSource, connectionString) + Migrate, err = migrate.New(DBMigrationsSource, ConnectionString) } if err != nil { die("Starting Migrate: " + err.Error()) diff --git a/traffic_ops_db/test/docker/Dockerfile-db-admin b/traffic_ops_db/test/docker/Dockerfile-db-admin index 6f60481..03745c9 100644 --- a/traffic_ops_db/test/docker/Dockerfile-db-admin +++ b/traffic_ops_db/test/docker/Dockerfile-db-admin @@ -23,25 +23,43 @@ FROM centos:7 ARG POSTGRES_VERSION=13.2 ENV POSTGRES_VERSION=$POSTGRES_VERSION -# NOTE: temporary workaround for removal of golang packages from CentOS 7 base repo RUN yum install -y \ - epel-release \ - centos-release-scl-rh \ - https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm \ + epel-release \ + centos-release-scl-rh \ + https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm \ git && \ - yum -y install golang + yum -y install \ + cpanminus \ + expat-devel \ + libcap \ + libcurl-devel \ + libidn-devel \ + libpcap-devel \ + mkisofs \ + openssl-devel \ + perl-core \ + perl-Crypt-ScryptKDF \ + perl-DBD-Pg \ + perl-DBI \ + perl-Digest-SHA1 \ + perl-JSON \ + perl-libwww-perl \ + perl-TermReadKey \ + perl-Test-CPAN-Meta \ + perl-WWW-Curl \ + postgresql13 \ + postgresql13-devel &&\ + yum clean all # Override TRAFFIC_OPS_RPM arg to use a different one using --build-arg TRAFFIC_OPS_RPM=... Can be local file or http://... ARG TRAFFIC_OPS_RPM=traffic_ops.rpm ADD $TRAFFIC_OPS_RPM / -RUN yum install -y \ - /$(basename $TRAFFIC_OPS_RPM) && \ - rm /$(basename $TRAFFIC_OPS_RPM) && \ - yum clean all +RUN rpm -Uvh $(basename $TRAFFIC_OPS_RPM) && \ + rm $(basename $TRAFFIC_OPS_RPM) WORKDIR /opt/traffic_ops/app -ADD run-db-test.sh \ +COPY run-db-test.sh \ db-config.sh \ / diff --git a/traffic_ops_db/test/docker/run-db-test.sh b/traffic_ops_db/test/docker/run-db-test.sh index ff476d6..3bbcb59 100755 --- a/traffic_ops_db/test/docker/run-db-test.sh +++ b/traffic_ops_db/test/docker/run-db-test.sh @@ -132,7 +132,7 @@ fi # test full restoration of the initial DB dump for d in $(get_db_dumps); do echo "testing restoration of DB dump: $d" - dropdb --echo --if-exists < "$d" > /dev/null || echo "Dropping DB ${DB_NAME} failed: $d" + dropdb --echo --if-exists "$DB_NAME" < "$d" > /dev/null || echo "Dropping DB ${DB_NAME} failed: $d" createdb --echo < "$d" > /dev/null || echo "Creating DB ${DB_NAME} failed: $d" pg_restore --verbose --clean --if-exists --exit-on-error -d "$DB_NAME" < "$d" > /dev/null || { echo "DB restoration failed: $d"; exit 1; } done
