Here is a changeset that adds a new puppet application to the puppet application portfolio: puppetcleaner.
This application removes all traces of a node on the puppetmaster (including certs, cached facts and nodes, reports, and storedconfig entries). Furthermore it is capable of unexporting exported resources of a host so that consumers of these resources can remove the exported resources and we will safely remove the node from our infrastructure. Usage: puppet clean [-u] <host> [<host2> ...] To achieve this we add different destroy methods to the different parts of the indirector. So for example for yaml indirections we already offer read access for the yaml, this changeset adds the destroy handler which only removes the yaml file for a request. This can be used to remove cached entries. This work is based on the initial work of Brice Figureau <[email protected]> Signed-off-by: Peter Meier <[email protected]> --- lib/puppet/application/clean.rb | 141 +++++++++++++ lib/puppet/indirector/report/processor.rb | 43 +++-- lib/puppet/indirector/yaml.rb | 5 + lib/puppet/reports/store.rb | 15 ++ lib/puppet/util/command_line.rb | 3 +- lib/puppet/util/command_line/clean | 58 ++++++ spec/unit/application/clean_spec.rb | 270 +++++++++++++++++++++++++ spec/unit/indirector/report/processor_spec.rb | 31 +++- spec/unit/indirector/yaml_spec.rb | 18 ++ 9 files changed, 561 insertions(+), 23 deletions(-) create mode 100644 lib/puppet/application/clean.rb create mode 100644 lib/puppet/util/command_line/clean create mode 100644 spec/unit/application/clean_spec.rb diff --git a/lib/puppet/application/clean.rb b/lib/puppet/application/clean.rb new file mode 100644 index 0000000..da94f4d --- /dev/null +++ b/lib/puppet/application/clean.rb @@ -0,0 +1,141 @@ +require 'puppet/application' + +class Puppet::Application::Clean < Puppet::Application + + should_parse_config + run_mode :master + + attr_reader :nodes + + option("--unexport","-u") + + def main + raise "At least one node should be passed" if nodes.empty? + begin + nodes.each do |node| + self.clean_cert(node) + self.clean_cached_facts(node) + self.clean_cached_node(node) + self.clean_reports(node) + self.clean_storeconfigs(node) + end + rescue => detail + puts detail.backtrace if Puppet[:trace] + puts detail.to_s + end + end + + # clean signed cert for +host+ + def clean_cert(node) + if Puppet::SSL::Host.ca_location == :local + ca.apply(:revoke, :to => [node]) + ca.apply(:destroy, :to => [node]) + Puppet.info "%s certificates removed from ca" % node + else + Puppet.info "Not managing %s certs as this host is not a CA" % node + end + end + + # clean facts for +host+ + def clean_cached_facts(node) + Puppet::Node::Facts.destroy(node) + Puppet.info "%s's facts removed" % node + end + + # clean cached node +host+ + def clean_cached_node(node) + Puppet::Node.destroy(node) + Puppet.info "%s's cached node removed" % node + end + + # clean node reports for +host+ + def clean_reports(node) + Puppet::Transaction::Report.destroy(node) + Puppet.info "%s's reports removed" % node + end + + # clean store config for +host+ + def clean_storeconfigs(node) + return unless Puppet[:storeconfigs] && Puppet.features.rails? + Puppet::Rails.connect + unless rails_node = Puppet::Rails::Host.find_by_name(node) + Puppet.notice "No entries found for %s in storedconfigs." % node + return + end + + if options[:unexport] + unexport(rails_node) + Puppet.notice "Force %s's exported resources to absent" % node + Puppet.warning "Please wait other host have checked-out their configuration before finishing clean-up wih:" + Puppet.warning "$ puppetclean #{node}" + else + rails_node.destroy + Puppet.notice "%s storeconfigs removed" % node + end + end + + def unexport(node) + # fetch all exported resource + query = {:include => {:param_values => :param_name}} + query[:conditions] = ["exported=? AND host_id=?", true, node.id] + + Puppet::Rails::Resource.find(:all, query).each do |resource| + if type_is_ensurable(resource) + line = 0 + param_name = Puppet::Rails::ParamName.find_or_create_by_name("ensure") + + if ensure_param = resource.param_values.find(:first, :conditions => [ 'param_name_id = ?', param_name.id]) + line = ensure_param.line.to_i + Puppet::Rails::ParamValue.delete(ensure_param.id); + end + + # force ensure parameter to "absent" + resource.param_values.create(:value => "absent", + :line => line, + :param_name => param_name) + Puppet.info("%s has been marked as \"absent\"" % resource.name) + end + end + end + + def setup + super + Puppet::Util::Log.newdestination(:console) + + Puppet.parse_config + + if options[:debug] + Puppet::Util::Log.level = :debug + elsif options[:verbose] + Puppet::Util::Log.level = :info + end + + @nodes = command_line.args.collect { |h| h.downcase } + + if Puppet::SSL::CertificateAuthority.ca? + Puppet::SSL::Host.ca_location = :local + else + Puppet::SSL::Host.ca_location = :none + end + + Puppet::Node::Facts.terminus_class = :yaml + Puppet::Node::Facts.cache_class = :yaml + Puppet::Node.terminus_class = :yaml + Puppet::Node.cache_class = :yaml + end + + private + def ca + @ca ||= Puppet::SSL::CertificateAuthority.instance + end + + def environment + @environemnt ||= Puppet::Node::Environment.new + end + + def type_is_ensurable(resource) + (type=Puppet::Type.type(resource.restype)) && type.validattr?(:ensure) || \ + (type = environment.known_resource_types.find_definition('',resource.restype)) && type.arguments.keys.include?('ensure') + end + +end \ No newline at end of file diff --git a/lib/puppet/indirector/report/processor.rb b/lib/puppet/indirector/report/processor.rb index 88fe4b4..8fbc4f7 100644 --- a/lib/puppet/indirector/report/processor.rb +++ b/lib/puppet/indirector/report/processor.rb @@ -14,28 +14,28 @@ class Puppet::Transaction::Report::Processor < Puppet::Indirector::Code process(request.instance) end + def destroy(request) + processors do |mod| + mod.destroy(request.key) if mod.respond_to?(:destroy) + end + end + private # Process the report with each of the configured report types. # LAK:NOTE This isn't necessarily the best design, but it's backward # compatible and that's good enough for now. def process(report) - return if Puppet[:reports] == "none" - - reports.each do |name| - if mod = Puppet::Reports.report(name) - # We have to use a dup because we're including a module in the - # report. - newrep = report.dup - begin - newrep.extend(mod) - newrep.process - rescue => detail - puts detail.backtrace if Puppet[:trace] - Puppet.err "Report #{name} failed: #{detail}" - end - else - Puppet.warning "No report named '#{name}'" + processors do |mod| + # We have to use a dup because we're including a module in the + # report. + newrep = report.dup + begin + newrep.extend(mod) + newrep.process + rescue => detail + puts detail.backtrace if Puppet[:trace] + Puppet.err "Report #{name} failed: #{detail}" end end end @@ -45,4 +45,15 @@ class Puppet::Transaction::Report::Processor < Puppet::Indirector::Code # LAK:NOTE See http://snurl.com/21zf8 [groups_google_com] x = Puppet[:reports].gsub(/(^\s+)|(\s+$)/, '').split(/\s*,\s*/) end + + def processors(&blk) + return if Puppet[:reports] == "none" + reports.each do |name| + if mod = Puppet::Reports.report(name) + yield(mod) + else + Puppet.warning "No report named '#{name}'" + end + end + end end diff --git a/lib/puppet/indirector/yaml.rb b/lib/puppet/indirector/yaml.rb index 23997e9..7b12d25 100644 --- a/lib/puppet/indirector/yaml.rb +++ b/lib/puppet/indirector/yaml.rb @@ -47,6 +47,11 @@ class Puppet::Indirector::Yaml < Puppet::Indirector::Terminus File.join(base, self.class.indirection_name.to_s, name.to_s + ext) end + def destroy(request) + file_path = path(request.key) + File.unlink(file_path) if File.exists?(file_path) + end + def search(request) Dir.glob(path(request.key,'')).collect do |file| YAML.load_file(file) diff --git a/lib/puppet/reports/store.rb b/lib/puppet/reports/store.rb index 99a9fc1..bc20773 100644 --- a/lib/puppet/reports/store.rb +++ b/lib/puppet/reports/store.rb @@ -38,5 +38,20 @@ Puppet::Reports.register_report(:store) do # Only testing cares about the return value file end + + # removes all reports for a given host + def self.destroy(host) + client = host.gsub("..",".") + dir = File.join(Puppet[:reportdir], client) + + if File.exists?(dir) + Dir.entries(dir).each do |file| + next if ['.','..'].include?(file) + file = File.join(dir, file) + File.unlink(file) if File.file?(file) + end + Dir.rmdir(dir) + end + end end diff --git a/lib/puppet/util/command_line.rb b/lib/puppet/util/command_line.rb index 3562a3d..0535a4b 100644 --- a/lib/puppet/util/command_line.rb +++ b/lib/puppet/util/command_line.rb @@ -2,7 +2,7 @@ module Puppet module Util class CommandLine - LegacyName = Hash.new{|h,k| k}.update( + LegacyName = Hash.new{|h,k| k}.update( { 'agent' => 'puppetd', 'cert' => 'puppetca', @@ -14,6 +14,7 @@ module Puppet 'resource' => 'ralsh', 'kick' => 'puppetrun', 'master' => 'puppetmasterd', + 'clean' => 'clean', }) diff --git a/lib/puppet/util/command_line/clean b/lib/puppet/util/command_line/clean new file mode 100644 index 0000000..c32bcfe --- /dev/null +++ b/lib/puppet/util/command_line/clean @@ -0,0 +1,58 @@ +#!/usr/bin/env ruby + +# +# = Synopsis +# +# Fully clean a puppetmaster view of a host +# +# = Usage +# +# puppet clean [-h|--help] [-d|--debug] [-v|--verbose] [-u|--unexport] host +# +# = Description +# +# This command cleans everything a puppetmaster knows about a node, including +# +# * Signed certificates ($vardir/ssl/ca/signed/node.domain.pem) +# * Cached facts ($vardir/yaml/facts/node.domain.yaml) +# * Cached node stuff ($vardir/yaml/node/node.domain.yaml) +# * Reports ($vardir/reports/node.domain) +# * Stored configs: it can either remove all data from an host in your storedconfig +# database, or with --unexport turn every exported resource supporting ensure to absent +# so that any other host checking out their config can remove those exported configurations. +# +# = Options +# +# Note that any configuration parameter that's valid in the configuration file +# is also a valid long argument. For example, 'ssldir' is a valid configuration +# parameter, so you can specify '--ssldir <directory>' as an argument. +# +# See the configuration file documentation at +http://docs.puppetlabs.com/references/latest/configuration.html for +# the full list of acceptable parameters. A commented list of all +# configuration options can also be generated by running puppet with +# '--genconfig'. +# +# help: +# Print this help message. +# +# unexport:: +# Instead of removing all the storedconfig data, it changes all exported resource's ensure parameter +# of this host to "absent". Then it is necessary to wait that all other host have checked out their +# configuration to run again puppetclean whithout this option to finish the clean-up. +# +# +# = Example +# +# puppeclean some-random-host.reductivelabs.com +# +# = Author +# +# Brice Figureau / Peter Meier +# +# = Copyright +# +# Copyright (c) 2005 Puppet Labs, LLC +# Licensed under the GNU Public License + +#Puppet::Application[:clean].run diff --git a/spec/unit/application/clean_spec.rb b/spec/unit/application/clean_spec.rb new file mode 100644 index 0000000..6de9f37 --- /dev/null +++ b/spec/unit/application/clean_spec.rb @@ -0,0 +1,270 @@ +#!/usr/bin/env ruby + +require File.dirname(__FILE__) + '/../../spec_helper' + +require 'puppet/application/clean' + +describe Puppet::Application::Clean do + before :each do + @clean_app = Puppet::Application[:clean] + Puppet::Util::Log.stubs(:newdestination) + Puppet::Util::Log.stubs(:level=) + end + + it "should operate in master run_mode" do + @clean_app.class.run_mode.name.should equal(:master) + end + + it "should ask Puppet::Application to parse Puppet configuration file" do + @clean_app.should_parse_config?.should be_true + end + + it "should declare a main command" do + @clean_app.should respond_to(:main) + end + + describe "when handling options" do + it "should declare handle_unexport method" do + @clean_app.should respond_to(:handle_unexport) + end + + it "should store argument value when calling handle_unexport" do + @clean_app.options.expects(:[]=).with(:unexport, 'arg') + @clean_app.send(:handle_unexport, 'arg') + end + end + + describe "during setup" do + before :each do + Puppet::Log.stubs(:newdestination) + Puppet::Log.stubs(:level=) + Puppet.stubs(:parse_config) + Puppet::Node::Facts.stubs(:terminus_class=) + Puppet::Node.stubs(:cache_class=) + end + + it "should set console as the log destination" do + Puppet::Log.expects(:newdestination).with(:console) + + @clean_app.setup + end + + it "should set facts terminus and cache class to yaml" do + Puppet::Node::Facts.expects(:terminus_class=).with(:yaml) + Puppet::Node::Facts.expects(:cache_class=).with(:yaml) + + @clean_app.setup + end + + it "should set node cache as yaml" do + Puppet::Node.expects(:terminus_class=).with(:yaml) + Puppet::Node.expects(:cache_class=).with(:yaml) + + @clean_app.setup + end + + it "should handle all other arguments as hostnames" do + @clean_app.command_line.stubs(:args).returns(['foo1.example.com','foo2.example.com']) + @clean_app.setup + @clean_app.nodes.should == ['foo1.example.com','foo2.example.com'] + end + + it "should manage the certs if the host is a CA" do + Puppet::SSL::CertificateAuthority.stubs(:ca?).returns(true) + Puppet::SSL::Host.expects(:ca_location=).with(:local) + @clean_app.setup + end + + it "should not manage the certs if the host is not a CA" do + Puppet::SSL::CertificateAuthority.stubs(:ca?).returns(false) + Puppet::SSL::Host.expects(:ca_location=).with(:none) + @clean_app.setup + end + end + + describe "when running" do + + before :each do + @host = 'node' + Puppet.stubs(:info) + [ "cert", "cached_facts", "cached_node", "reports", "storeconfigs" ].each do |m| + @clean_app.stubs("clean_#{m}".to_sym).with(@host) + end + @clean_app.stubs(:nodes).returns([...@host]) + end + + it "should raise an error if no host is given" do + @clean_app.stubs(:nodes).returns([]) + lambda { @clean_app.main }.should raise_error + end + + [ "cert", "cached_facts", "cached_node", "reports", "storeconfigs" ].each do |m| + it "should clean #{m.sub('_',' ')}" do + @clean_app.expects("clean_#{m}".to_sym).with(@host) + + @clean_app.main + end + end + + end + + describe "when cleaning certificate" do + before :each do + Puppet::SSL::Host.stubs(:destroy) + @ca = mock() + Puppet::SSL::CertificateAuthority.stubs(:instance).returns(@ca) + end + + it "should send the :destroy order to the ca if we are a CA" do + Puppet::SSL::Host.stubs(:ca_location).returns(:local) + @ca.expects(:apply).with { |cert_mode,to| cert_mode == :revoke } + @ca.expects(:apply).with { |cert_mode,to| cert_mode == :destroy } + + @clean_app.clean_cert(@host) + end + + it "should not destroy the certs if we are not a CA" do + Puppet::SSL::Host.stubs(:ca_location).returns(:none) + @ca.expects(:apply).never + @clean_app.clean_cert(@host) + end + end + + describe "when cleaning cached facts" do + it "should destroy facts" do + @host = 'node' + Puppet::Node::Facts.expects(:destroy).with(@host) + + @clean_app.clean_cached_facts(@host) + end + end + + describe "when cleaning cached node" do + it "should destroy the cached node" do + Puppet::Node::Yaml.any_instance.expects(:destroy) + @clean_app.clean_cached_node(@host) + end + end + + describe "when cleaning archived reports" do + it "should tell the reports to remove themselves" do + Puppet::Transaction::Report.stubs(:destroy).with(@host) + + @clean_app.clean_reports(@host) + end + end + + describe "when cleaning storeconfigs entries for host", :if => Puppet.features.rails? do + before :each do + # Stub this so we don't need access to the DB. + require 'puppet/rails/host' + + @clean_app.options.stubs(:[]).with(:unexport).returns(false) + Puppet.stubs(:[]).with(:storeconfigs).returns(true) + + Puppet::Rails.stubs(:connect) + @rails_node = stub_everything 'rails_node' + Puppet::Rails::Host.stubs(:find_by_name).returns(@rails_node) + @clean_app.stubs(:destroy) + end + + it "should connect to the database" do + Puppet::Rails.expects(:connect) + @clean_app.clean_storeconfigs(@host) + end + + it "should find the right host entry" do + Puppet::Rails::Host.expects(:find_by_name).with(@host).returns(@rails_node) + + @clean_app.clean_storeconfigs(@host) + end + + describe "without unexport" do + it "should remove the host and it's content" do + @rails_node.expects(:destroy) + + @clean_app.clean_storeconfigs(@host) + end + end + + describe "with unexport" do + before :each do + @clean_app.options.stubs(:[]).with(:unexport).returns(true) + @rails_node.stubs(:id).returns(1234) + + @type = stub_everything 'type' + @type.stubs(:validattr?).with(:ensure).returns(true) + + @ensure_name = stub_everything 'ensure_name', :id => 23453 + Puppet::Rails::ParamName.stubs(:find_or_create_by_name).returns(@ensure_name) + + @param_values = stub_everything 'param_values' + @resource = stub_everything 'resource', :param_values => @param_values, :restype => "File" + Puppet::Rails::Resource.stubs(:find).returns([...@resource]) + end + + it "should find all resources" do + Puppet::Rails::Resource.expects(:find).with(:all, {:include => {:param_values => :param_name}, :conditions => ["exported=? AND host_id=?", true, 1234]}).returns([]) + + @clean_app.clean_storeconfigs(@host) + end + + describe "with an exported native type" do + before :each do + Puppet::Type.stubs(:type).returns(@type) + @type.expects(:validattr?).with(:ensure).returns(true) + end + + it "should test a native type for ensure as an attribute" do + @clean_app.clean_storeconfigs(@host) + end + + it "should delete the old ensure parameter" do + ensure_param = stub 'ensure_param', :id => 12345, :line => 12 + @param_values.stubs(:find).returns(ensure_param) + Puppet::Rails::ParamValue.expects(:delete).with(12345); + @clean_app.clean_storeconfigs(@host) + end + + it "should add an ensure => absent parameter" do + @param_values.expects(:create).with(:value => "absent", + :line => 0, + :param_name => @ensure_name) + @clean_app.clean_storeconfigs(@host) + end + end + + describe "with an exported definition" do + it "should try to lookup a definition and test it for the ensure argument" do + Puppet::Type.stubs(:type).returns(nil) + definition = stub_everything 'definition', :arguments => { 'ensure' => 'present' } + Puppet::Resource::TypeCollection.any_instance.expects(:find_definition).with('',"File").returns(definition) + @clean_app.clean_storeconfigs(@host) + end + end + + it "should not unexport the resource of an unkown type" do + Puppet::Type.stubs(:type).returns(nil) + Puppet::Resource::TypeCollection.any_instance.expects(:find_definition).with('',"File").returns(nil) + Puppet::Rails::ParamName.expects(:find_or_create_by_name).never + @clean_app.clean_storeconfigs(@host) + end + + it "should not unexport the resource of a not ensurable native type" do + Puppet::Type.stubs(:type).returns(@type) + @type.expects(:validattr?).with(:ensure).returns(false) + Puppet::Resource::TypeCollection.any_instance.expects(:find_definition).with('',"File").returns(nil) + Puppet::Rails::ParamName.expects(:find_or_create_by_name).never + @clean_app.clean_storeconfigs(@host) + end + + it "should not unexport the resource of a not ensurable definition" do + Puppet::Type.stubs(:type).returns(nil) + definition = stub_everything 'definition', :arguments => { 'foobar' => 'someValue' } + Puppet::Resource::TypeCollection.any_instance.expects(:find_definition).with('',"File").returns(definition) + Puppet::Rails::ParamName.expects(:find_or_create_by_name).never + @clean_app.clean_storeconfigs(@host) + end + end + end +end diff --git a/spec/unit/indirector/report/processor_spec.rb b/spec/unit/indirector/report/processor_spec.rb index 5602a27..76420df 100755 --- a/spec/unit/indirector/report/processor_spec.rb +++ b/spec/unit/indirector/report/processor_spec.rb @@ -15,26 +15,45 @@ describe Puppet::Transaction::Report::Processor do it "should provide a method for saving reports" do Puppet::Transaction::Report::Processor.new.should respond_to(:save) end + + it "should provide a method for cleaning reports" do + Puppet::Transaction::Report::Processor.new.should respond_to(:destroy) + end + end -describe Puppet::Transaction::Report::Processor, " when saving a report" do +describe Puppet::Transaction::Report::Processor, " when processing a report" do before do Puppet.settings.stubs(:use) @reporter = Puppet::Transaction::Report::Processor.new + @request = stub 'request', :instance => mock("report"), :key => 'node' end - it "should not process the report if reports are set to 'none'" do + it "should not save the report if reports are set to 'none'" do Puppet::Reports.expects(:report).never Puppet.settings.expects(:value).with(:reports).returns("none") - request = stub 'request', :instance => mock("report") - - @reporter.save(request) + @reporter.save(@request) end - it "should process the report with each configured report type" do + it "should save the report with each configured report type" do Puppet.settings.stubs(:value).with(:reports).returns("one,two") @reporter.send(:reports).should == %w{one two} + + Puppet::Reports.expects(:report).with('one') + Puppet::Reports.expects(:report).with('two') + + @reporter.save(@request) + end + + it "should destroy reports for each processor that responds to destroy" do + Puppet.settings.stubs(:value).with(:reports).returns("http,store") + http_report = mock() + store_report = mock() + store_report.expects(:destroy).with(@request.key) + Puppet::Reports.expects(:report).with('http').returns(http_report) + Puppet::Reports.expects(:report).with('store').returns(store_report) + @reporter.destroy(@request) end end diff --git a/spec/unit/indirector/yaml_spec.rb b/spec/unit/indirector/yaml_spec.rb index 86c13c5..e193727 100755 --- a/spec/unit/indirector/yaml_spec.rb +++ b/spec/unit/indirector/yaml_spec.rb @@ -154,5 +154,23 @@ describe Puppet::Indirector::Yaml, " when choosing file location" do Dir.expects(:glob).with(:glob).returns [] @store.search(@request).should == [] end + + describe Puppet::Indirector::Yaml, " when destroying" do + it "should unlink the right yaml file if it exists" do + path = File.join("/what/ever", @store.class.indirection_name.to_s, @request.key.to_s + ".yaml") + File.expects(:exists?).with(path).returns true + File.expects(:unlink).with(path) + + @store.destroy(@request) + end + + it "should not unlink the yaml file if it does not exists" do + path = File.join("/what/ever", @store.class.indirection_name.to_s, @request.key.to_s + ".yaml") + File.expects(:exists?).with(path).returns false + File.expects(:unlink).with(path).never + + @store.destroy(@request) + end + end end end -- 1.7.3.4 -- You received this message because you are subscribed to the Google Groups "Puppet Developers" group. To post to this group, send email to [email protected]. To unsubscribe from this group, send email to [email protected]. For more options, visit this group at http://groups.google.com/group/puppet-dev?hl=en.
