On Wed, Aug 15, 2012 at 02:12:45PM -0500, Steve Linabery wrote: > Brought in and refactored katello-upgrade[1] to more closely follow their > system of handling upgrades. > > This script and associate files provide us with: > 1) better logging of upgrade process (previously limited to watching stdout) > 2) history of what upgrade scripts have been applied > 3) directory where we can drop incremental upgrade scripts for > more modular upgrade control. > 4) requires user to stop services manually or use --auto-stop option, thus > avoiding unintentional data loss in currently running services > > [1] https://github.com/Katello/katello/blob/master/puppet/bin/katello-upgrade > --- > aeolus-conductor.spec.in | 3 + > src/script/aeolus-upgrade | 479 > ++++++++++++++++++++ > src/script/upgrade | 7 - > .../default/0001_migrate_conductor_db.sh | 27 ++ > .../default/0002_run_aeolus-configure.sh | 13 + > src/upgrade-scripts/upgrade/0001_data_defn.sh | 28 ++ > src/upgrade-scripts/upgrade/0002_example_script.py | 17 + > 7 files changed, 567 insertions(+), 7 deletions(-) > create mode 100755 src/script/aeolus-upgrade > delete mode 100755 src/script/upgrade > create mode 100755 src/upgrade-scripts/default/0001_migrate_conductor_db.sh > create mode 100755 src/upgrade-scripts/default/0002_run_aeolus-configure.sh > create mode 100755 src/upgrade-scripts/upgrade/0001_data_defn.sh > create mode 100755 src/upgrade-scripts/upgrade/0002_example_script.py > > diff --git a/aeolus-conductor.spec.in b/aeolus-conductor.spec.in > index 0eb5c4f..0354d92 100644 > --- a/aeolus-conductor.spec.in > +++ b/aeolus-conductor.spec.in > @@ -353,6 +353,7 @@ touch %{buildroot}%{app_root}/log/delayed_job.log > > # copy script files over > %{__cp} -r src/script %{buildroot}%{app_root} > +%{__cp} -r src/upgrade-scripts %{buildroot}%{app_root} > > %{__mkdir} -p %{buildroot}%{_sysconfdir}/%{name} > > @@ -458,6 +459,8 @@ fi > %config %{_sysconfdir}/%{name} > %doc AUTHORS COPYING > %{app_root}/script/upgrade > +%{app_root}/script/aeolus-upgrade > +%{app_root}/upgrade-scripts > > %files daemons > %{_initrddir}/aeolus-conductor > diff --git a/src/script/aeolus-upgrade b/src/script/aeolus-upgrade > new file mode 100755 > index 0000000..2897f4a > --- /dev/null > +++ b/src/script/aeolus-upgrade > @@ -0,0 +1,479 @@ > +#!/usr/bin/ruby > + > +require 'optparse' > + > + > +AEOLUS_UPGRADE_DIR = ENV['AEOLUS_UPGRADE_DIR'] || > '/usr/share/aeolus-conductor/upgrade-scripts/' > +HISTORY_FILE_PATH = ENV['AEOLUS_UPGRADE_HISTORY'] || > '/var/lib/aeolus-conductor/upgrade-history' > +DEFAULT_LOG_FILE = '/var/log/aeolus-conductor/aeolus_upgrade.log' > +DEFAULT_SCRIPTS_DIR = 'default/' > +UPGRADE_SCRIPTS_DIR = 'upgrade/' > + > +# error codes for exit_with function > +ERROR_CODES = { > + :success => 0, > + :general_error => 1, > + :option_parser_error => 2, > + :not_root => 3, > + :io_error => 4, > + :unknown => 127, > +} > + > +BACKEND_SERVICES = ["mongod", > + "iwhd", > + "postgresql", > + "httpd", > + "deltacloud-core", > + "libvirtd", > + "aeolus-conductor", > + "conductor-dbomatic", > + "imagefactory", > + "ntpd"] > +STDOUT_NAME = "stdout" > + > +# Terminate script with error code from ERROR_CODES hash > +def exit_with(code = :unknown) > + code = ERROR_CODES[code.to_sym] || ERROR_CODES[:unknown] > + exit code > +end > + > +# Indent each line of text with spacing > +def indent_text(text, spacing=" ") > + text.gsub(/^/, spacing) > +end > + > +# Prints message and waits for Y/n answer. > +# Returns true if the answer is 'Y', false if it is 'n'. > +def confirm(message) > + yes = 'y' > + no = 'n' > + begin > + print message + " (y/n): " > + answer = gets.strip > + end while !(answer == yes or answer == no) > + return answer == yes > +end > + > + > +# Stores history of executed scripts > +class UpgradeHistory > + > + def initialize(path_to_history_file) > + @history_file = path_to_history_file > + end > + > + # Returns last inserted executed script > + def get_last_script > + last_line = nil > + f = open_history_file 'r' > + f.each_line do |line| > + last_line = line > + end > + f.close > + last_line > + end > + > + # Ddds a script to history > + def add_script(script) > + f = open_history_file 'a' > + f.puts File.basename(script) > + f.close > + end > + > + private > + > + def ensure_file(path) > + if not File.exists? path > + f = File.new(path, "w") > + f.close > + end > + end > + > + def open_history_file(mode) > + ensure_file @history_file > + File.open(@history_file, mode) > + rescue => e > + $stderr.puts e.message > + exit_with :io_error > + end > + > +end > + > + > +# Dummy history with /dev/null-like behaviour > +class DevNullHistory > + > + def get_last_script > + nil > + end > + > + def add_script(script) > + end > +end > + > + > +# Queue of upgrade scripts. > +class UpgradeQueue > + > + # Load scripts form a directory > + # Takes path to a directory with upgrade scripts, an upgrade history > instance > + # and an instance of script parser. > + # The queue finds all scripts applicable to current deployment that have > not been > + # executed yet. > + def load(script_dir, history, script_parser) > + @history = history > + @script_dir = script_dir > + @script_parser = script_parser > + @scripts = nil > + end > + > + # Returns all scripts in the queue that have not been executed yet > + def get_scripts > + return [] if not @history or not @script_dir > + > + @scripts ||= upgrade_scripts(@script_dir, @history.get_last_script) > + @scripts > + end > + > + # Process all scripts in the queue. Execute a block for each of them > + def process &block > + while not self.get_scripts.empty? > + yield self.get_scripts.first, @history > + self.get_scripts.shift > + end > + end > + > + def empty? > + self.get_scripts.empty? > + end > + > + def length > + self.get_scripts.length > + end > + > + protected > + > + def upgrade_scripts(dir_path, starting_from = nil) > + script_infos = find_upgrade_scripts(dir_path, starting_from).map do > |path| > + @script_parser.get_info(path) > + end > + > + return filter_scripts_for("aeolus-conductor", script_infos) > + end > + > + def filter_scripts_for(deployment, script_infos) > + script_infos.delete_if do |script| > + not script[:apply].include? deployment > + end > + end > + > + def find_upgrade_scripts(path, starting_from = nil) > + scripts = Dir.glob(path+'*').select{|f| File.executable?(f)}.sort > + if not starting_from.nil? > + scripts = scripts.delete_if {|script| (script <= path+starting_from)} > + end > + return scripts > + end > + > +end > + > +# Parses upgrade scripts headers > +# The files can contain: > +# '#name: NAME' - on a single line, pretty name of the script > +# '#description: DESC' - can be multiline, longer description of the script > +class ScriptParser > + > + # Returns info available from the file header > + def get_info(script_path) > + @info = {} > + parse_script_file script_path > + @info[:name] ||= "" > + @info[:apply] ||= [] > + @info[:filename] = File.basename(script_path) > + @info[:path] = script_path > + @info > + end > + > + private > + > + def parse_script_file(script_path) > + f = File.open(script_path, 'r') > + f.each_line do |line| > + process_line line > + end > + f.close > + end > + > + def process_line(line) > + @in_desc ||= false > + > + if line =~ /#\s*name:\s*(.*)\s*$/ > + @info[:name] = $1 > + @in_desc = false > + > + elsif line =~ /#\s*apply:\s*(.*)$/ > + @info[:apply] = $1.strip.split(/\s/) > + @in_desc = false > + > + elsif /#\s*description:\s*(.*)\s*$/.match line > + @info[:description] = $1 > + @in_desc = true > + > + elsif ( line =~ /^\s*#.*/ ) and @in_desc > + @info[:description] ||= "" > + @info[:description] += "\n" + line.gsub(/\s*#/, "").strip > + @info[:description] = @info[:description].strip > + > + else > + @in_desc = false > + > + end > + end > + > +end > + > + > +# The upgrade process itself. > +# It creates two upgrade queues: > +# - one for default scripts that are executed everytime > +# - one for one-time scripts that get executed only once > +# Tries to execute the scripts one by one. If any of them fails, the process > +# is stopped. > +# Prints a result status at the end. > +class UpgradeProcess > + > + LINE_LEN = 80 > + > + def run(options) > + @options = options > + > + # check if backend services are stopped > + ensure_services_are_off > + > + fake_history = DevNullHistory.new > + upgrade_history = UpgradeHistory.new(HISTORY_FILE_PATH) > + @default_queue = UpgradeQueue.new > + @default_queue.load(AEOLUS_UPGRADE_DIR+DEFAULT_SCRIPTS_DIR, > fake_history, ScriptParser.new) if not @options[:skip_default] > + @upgrade_queue = UpgradeQueue.new > + @upgrade_queue.load(AEOLUS_UPGRADE_DIR+UPGRADE_SCRIPTS_DIR, > upgrade_history, ScriptParser.new) > + > + print_header > + > + @step_count = @upgrade_queue.length > + @step_count += @default_queue.length > + > + if @step_count == 0 > + print_nothing_to_do > + exit_with :success > + end > + > + process_queue(@default_queue) > + process_queue(@upgrade_queue) if @default_queue.empty? > + > + print_result > + > + if finished_step_count < @step_count > + exit_with :general_error > + else > + exit_with :success > + end > + end > + > + protected > + > + def ensure_services_are_off > + stopped = true > + BACKEND_SERVICES.each do |service| > + if !is_stopped? service > + puts "Service '%s' can not be running while aeolus-upgrade is in > progress" % service > + stopped = false > + end > + end > + if !stopped > + print_line > + if @options[:auto_stop] > + stop_services > + else > + puts "This script can be run in auto-stop mode with the -a / > --auto-stop option to" > + puts "disable services automatically." > + puts "You may always use `service aeolus-services stop` to stop all > the services" > + puts "" > + exit_service_stop > + end > + end > + end > + > + def stop_services > + service = 'aeolus-services' > + puts "We will stop all aeolus services using #{service} stop." > + if confirm("PROCEED?") > + if !stop(service) > + puts "There was an issue stopping your services. Please resolve this > manually" > + puts "and try again" > + exit_with :general_error > + end > + else > + exit_service_stop > + end > + end > + > + def exit_service_stop > + puts "Exiting. Please stop your services and try again" > + exit_with :general_error > + end > + > + def print_line(char="=", repeats=LINE_LEN) > + puts char*repeats > + end > + > + def print_nothing_to_do > + print_line > + puts "Nothing to do" > + end > + > + def print_header > + print_line > + puts " Aeolus-conductor upgrade" > + end > + > + def print_step_info info > + print_line > + puts > + puts "%s/%s: %s (%s)" % [@current_step, @step_count, info[:name], > info[:filename]] > + puts indent_text(info[:description]) if info[:description] > + puts > + end > + > + def print_result > + print_line > + puts "Upgrade successful" if finished_step_count == @step_count > + puts "Finished %i of %i upgrade steps" % [finished_step_count, > @step_count] > + end > + > + def finished_step_count > + @step_count - @upgrade_queue.length - @default_queue.length > + end > + > + def process_queue(queue) > + @current_step ||= 1 > + queue.process do |script_info, history| > + print_step_info(script_info) > + > + if not @options[:dry_run] > + break if not @options[:assume_yes] and not confirm("Do you want to > proceed?") > + break if not process_script(script_info) > + > + history.add_script script_info[:path] > + end > + > + @current_step+=1 > + end > + end > + > + def process_script(script_info) > + if execute_script(script_info[:path]) > + puts > + puts script_info[:name]+" OK." > + puts > + return true > + else > + puts > + puts script_info[:name]+" FAILED." > + puts > + return false > + end > + end > + > + def execute_script script > + command = script > + command += ' &>>'+ log_file if log_file != STDOUT_NAME > + > + log_start(script) if log_file != STDOUT_NAME > + result = system(command) > + log_result(result) if log_file != STDOUT_NAME > + > + return result > + end > + > + def log_start script > + system('printf "\n[$(date)] '+ script +'\n" >> '+ log_file) > + end > + > + def log_result success > + system('printf "SUCCEEDED\n" >> '+ log_file) if success > + system('printf "FAILED\n" >> '+ log_file) if not success > + end > + > + def log_file > + return @options[:log_file] || DEFAULT_LOG_FILE > + end > + > +end > + > +# test if given service is stopped > +def is_stopped? (service) > + return !system('/sbin/service %s status 2>/dev/null >/dev/null' % service) > +end > + > +# system appears to return TRUE in some cases even when the > +# system call fails. non catastrophic errors, such as the service > +# already being off > +def stop(service) > + return system('/sbin/service %s stop' % service) > +end > + > +# Parse and return script options > +def parse_options > + opts = {} > + opts[:assume_yes] = false > + > + begin > + option_parser = OptionParser.new > + option_parser.banner = "Usage: #{$0} [options]" > + > + option_parser.on_tail.on('-y', '--assumeyes', 'Assume yes on > confirmations') do > + opts[:assume_yes] = true > + end > + > + option_parser.on_tail.on('-a', '--autostop', 'Automatically stop > services') do > + opts[:auto_stop] = true > + end > + > + option_parser.on_tail.on('-d', '--dry-run', 'Prints the upgrade steps > without modifying anything') do > + opts[:dry_run] = true > + end > + > + option_parser.on_tail.on('-s', '--skip-default', 'Skips the default > upgrade steps') do > + opts[:skip_default] = true > + end > + > + option_parser.on_tail.on('--log=LOG_FILE', 'Log file, can also be set to > stdout') do |value| > + opts[:log_file] = value > + end > + > + option_parser.on_tail.on('-h', '--help', 'Show this short summary') do > + puts option_parser > + exit_with :success > + end > + > + option_parser.parse! > + rescue => e > + $stderr.puts e.message > + $stderr.puts option_parser > + exit_with :option_parser_error > + end > + opts > +end > + > + > +# check if running as root > +unless Process.uid == 0 > + $stderr.puts "You must run aeolus-upgrade as root" > + exit_with :not_root > +end > + > +# start the upgrade process > +upgrade = UpgradeProcess.new > +upgrade.run(parse_options) > + > + > diff --git a/src/script/upgrade b/src/script/upgrade > deleted file mode 100755 > index e3f658e..0000000 > --- a/src/script/upgrade > +++ /dev/null > @@ -1,7 +0,0 @@ > -#!/bin/sh > - > -cd /usr/share/aeolus-conductor > -aeolus-services stop > -service postgresql start > -RAILS_ENV=production rake dc:upgrade > -aeolus-services start > diff --git a/src/upgrade-scripts/default/0001_migrate_conductor_db.sh > b/src/upgrade-scripts/default/0001_migrate_conductor_db.sh > new file mode 100755 > index 0000000..70dfad0 > --- /dev/null > +++ b/src/upgrade-scripts/default/0001_migrate_conductor_db.sh > @@ -0,0 +1,27 @@ > +#!/bin/bash > + > +#name: Migrate aeolus-conductor rails database > +#apply: aeolus-conductor > +#description: Executes rake db:migrate for the aeolus-conductor database > + > +# default configuration values (should be the same as in our sysv init > script) > +if [ -f /etc/sysconfig/aeolus-conductor ]; then . > /etc/sysconfig/aeolus-conductor; fi > +AEOLUS_HOME=${AEOLUS_HOME:-/usr/share/aeolus-conductor} > +AEOLUS_ENV=${AEOLUS_ENV:-production} > + > +SERVICES="postgresql" > +for SERVICE in $SERVICES; do > + service $SERVICE start > +done > + > + > +pushd $AEOLUS_HOME >/dev/null > +RAILS_ENV=$AEOLUS_ENV rake db:migrate --trace 2>&1 > +ret_code=$? > +popd >/dev/null > + > +for SERVICE in $SERVICES; do > + service $SERVICE stop > +done > + > +exit $ret_code > diff --git a/src/upgrade-scripts/default/0002_run_aeolus-configure.sh > b/src/upgrade-scripts/default/0002_run_aeolus-configure.sh > new file mode 100755 > index 0000000..e84f1b5 > --- /dev/null > +++ b/src/upgrade-scripts/default/0002_run_aeolus-configure.sh > @@ -0,0 +1,13 @@ > +#!/bin/bash > + > +#name: Execute aeolus-configure > +#apply: aeolus-conductor > +#description: Executes aeolus-configure. Note: this restarts aeolus > services. Skip this step if you want to manually run aeolus-configure later. > + > +# default configuration values (should be the same as in our sysv init > script) > +if [ -f /etc/sysconfig/aeolus-conductor ]; then . > /etc/sysconfig/aeolus-conductor; fi > + > +aeolus-configure 2>&1 > +ret_code=$? > + > +exit $ret_code > diff --git a/src/upgrade-scripts/upgrade/0001_data_defn.sh > b/src/upgrade-scripts/upgrade/0001_data_defn.sh > new file mode 100755 > index 0000000..dfdb790 > --- /dev/null > +++ b/src/upgrade-scripts/upgrade/0001_data_defn.sh > @@ -0,0 +1,28 @@ > +#!/bin/bash > + > +#name: Update aeolus-conductor Data Definition > +#apply: aeolus-conductor > +#description: Adds a role definition to the existing seeded data definition. > noop if role is already defined. > + > +# default configuration values (should be the same as in our sysv init > script) > +if [ -f /etc/sysconfig/aeolus-conductor ]; then . > /etc/sysconfig/aeolus-conductor; fi > +AEOLUS_HOME=${AEOLUS_HOME:-/usr/share/aeolus-conductor} > +AEOLUS_ENV=${AEOLUS_ENV:-production} > + > +SERVICES="postgresql" > +for SERVICE in $SERVICES; do > + service $SERVICE start > +done > + > + > +pushd $AEOLUS_HOME >/dev/null > +#if data defn has been previously added, this is a noop > +RAILS_ENV=$AEOLUS_ENV rake dc:upgrade --trace 2>&1 > +ret_code=$? > +popd >/dev/null > + > +for SERVICE in $SERVICES; do > + service $SERVICE stop > +done > + > +exit $ret_code > diff --git a/src/upgrade-scripts/upgrade/0002_example_script.py > b/src/upgrade-scripts/upgrade/0002_example_script.py > new file mode 100755 > index 0000000..25b15ba > --- /dev/null > +++ b/src/upgrade-scripts/upgrade/0002_example_script.py > @@ -0,0 +1,17 @@ > +#!/usr/bin/python > + > +#name: Example script 2 > +#apply: katello > +#description: Empty python script > + > +import sys > + > +# TODO > +print "Test script output" > +print "Test script output 2" > +sys.stdout.flush() > + > +print >> sys.stderr, "Test script error output" > +sys.stderr.flush() > + > +exit(0) > \ No newline at end of file > -- > 1.7.7.6 >
Resending with whitespace fixes (!) and one-line change to specfile. s|e
