Giuseppe Lavagetto has submitted this change and it was merged. Change subject: scap: introduce scap_source type ......................................................................
scap: introduce scap_source type As scap::source is a mess of nested execs, which is hard to change without changing a few fundamental classes like git::clone, move to using a custom puppet type, which should also allow us to define multiple providers for various sources and add new providers for different sources if we need to. Advantages at the moment compared to scap::source: - It's faster - It cleanly checks if the git repo is present - It's absentable - It can be easily extended and could, in the future, also check that the origin url is correct and change it in case of need. Change-Id: If8ed793759ebce183ef7ec8f36decfd287438d4d --- A modules/scap/lib/puppet/provider/scap_source.rb A modules/scap/lib/puppet/provider/scap_source/gerrit.rb A modules/scap/lib/puppet/provider/scap_source/phabricator.rb A modules/scap/lib/puppet/type/scap_source.rb M modules/scap/manifests/server.pp 5 files changed, 357 insertions(+), 6 deletions(-) Approvals: Giuseppe Lavagetto: Verified; Looks good to me, approved diff --git a/modules/scap/lib/puppet/provider/scap_source.rb b/modules/scap/lib/puppet/provider/scap_source.rb new file mode 100644 index 0000000..9e01921 --- /dev/null +++ b/modules/scap/lib/puppet/provider/scap_source.rb @@ -0,0 +1,145 @@ +# Puppet base provider for type `scap_source`, which is needed to set up a +# base repository to use with the `scap3` deployment system +# +# Copyright (c) 2016 Giuseppe Lavagetto +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'puppet/provider' +require 'puppet/util/execution' +require 'fileutils' +require 'etc' + +# rubocop:disable ClassAndModuleCamelCase +class Puppet::Provider::Scap_source < Puppet::Provider + initvars + + has_command(:git, '/usr/bin/git') + + BASE_PATH = '/srv/deployment' + + # Shortand for the repo name + def repo + resource[:repository] + end + + # The origin of the repository + def origin(*) + raise Puppet::Error "origin must be implemented by actual providers" + end + + # The path to install the git clone to + def repo_path + if resource[:name].include?(File::SEPARATOR) + resource[:name] + else + File.join(resource[:name], resource[:name]) + end + end + + def target_path + path = File.expand_path(File.join(BASE_PATH, repo_path)) + + unless path.start_with?(BASE_PATH) + raise Puppet::Error, "Target path '#{path}' is invalid." + end + path + end + + def deploy_root + @deploy_root ||= File.dirname(target_path) + end + + def checkout(name, path) + umask = 022 + file_mode = 02775 + unless Dir.exists?(path) + FileUtils.makedirs path, :mode => file_mode + FileUtils.chown_R resource[:owner], resource[:group], path + end + pwd = Etc.getpwnam(resource[:owner]) + uid = pwd.uid + gid = pwd.gid + Puppet::Util.withumask( + umask) { + Puppet::Util::Execution.execute( + [ + self.class.command(:git), + '-c', 'core.sharedRepository=group', + 'clone', + '--recurse-submodules', + origin(name), + path, + ], + :uid => uid, + :gid => gid, + :failonfail => true, + ) + } + end + + def exists? + # check if the dirs exist; if they do, check + # if they're a git repo. + Puppet.debug("Checking existence of #{target_path}") + return false unless File.directory?(target_path) + # If the resource needs to be present, let's also check + # the dir is a git repo + if resource[:ensure] == :present + begin + git('-C', target_path, 'rev-parse', 'HEAD') + return true + rescue Puppet::ExecutionFailure + return false + end + else + return true + end + end + + def create + # Create the parent directory + unless Dir.exists?(deploy_root) + Puppet.debug("Creating #{deploy_root}") + FileUtils.makedirs deploy_root, :mode => 0755 + FileUtils.chown_R resource[:owner], resource[:group], deploy_root + end + + # Checkout the main repository, and the scap one too + Puppet.debug("Checking out #{repo} into #{target_path}") + checkout repo, target_path + Puppet.debug("Repository checked out in #{target_path}") + # rubocop:disable GuardClause + if resource[:scap_repository] + target = File.join(target_path, 'scap') + checkout resource[:scap_repository], target + end + end + + def destroy + Puppet.info("Deleting #{target_path} and all its empty parents") + dir = target_path + FileUtils.remove_entry_secure(dir, :force => true) + loop do + dir, _ = File.split(dir) + break if BASE_PATH.include?(dir) + begin + Dir.delete(dir) + rescue Errno::ENOTEMPTY + Puppet.info("Not removing #{dir} as it's not empty.") + break + end + end + end +end diff --git a/modules/scap/lib/puppet/provider/scap_source/gerrit.rb b/modules/scap/lib/puppet/provider/scap_source/gerrit.rb new file mode 100644 index 0000000..5395bab --- /dev/null +++ b/modules/scap/lib/puppet/provider/scap_source/gerrit.rb @@ -0,0 +1,35 @@ +# Puppet gerrit provider for type `scap_source`, which is needed to set up a +# base repository to use with the `scap3` deployment system +# +# Copyright (c) 2016 Giuseppe Lavagetto +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'puppet/provider/scap_source' + +Puppet::Type.type(:scap_source).provide(:gerrit, :parent => Puppet::Provider::Scap_source) do + desc 'Puppet provider for scap_source, for gerrit projects' + + # Gerrit is the default independently of system facts, so this little hack + def self.default? + true + end + + # The origin of the repository + def origin(repo_name) + "https://gerrit.wikimedia.org/r/p/#{repo_name}.git"; + end + + +end diff --git a/modules/scap/lib/puppet/provider/scap_source/phabricator.rb b/modules/scap/lib/puppet/provider/scap_source/phabricator.rb new file mode 100644 index 0000000..c7b889d --- /dev/null +++ b/modules/scap/lib/puppet/provider/scap_source/phabricator.rb @@ -0,0 +1,29 @@ +# Puppet phabricator provider for type `scap_source`, which is needed to set up a +# base repository to use with the `scap3` deployment system +# +# Copyright (c) 2016 Giuseppe Lavagetto +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'puppet/provider/scap_source' + +Puppet::Type.type(:scap_source).provide(:phabricator, :parent => Puppet::Provider::Scap_source) do + desc 'Puppet provider for scap_source, for gerrit projects' + + # The origin of the repository + def origin(repo_name) + "https://phabricator.wikimedia.org/diffusion/#{repo_name}.git"; + end + +end diff --git a/modules/scap/lib/puppet/type/scap_source.rb b/modules/scap/lib/puppet/type/scap_source.rb new file mode 100644 index 0000000..b57aeb1 --- /dev/null +++ b/modules/scap/lib/puppet/type/scap_source.rb @@ -0,0 +1,142 @@ +# == resource scap_source +# +# Sets up scap3 deployment source on a deploy server. +# This will clone $repository at /srv/deployment/$title. +# If $scap_repository is set it will clone it at +# /srv/deployment/$title/scap. If you set $scap_repository to true, +# this will assume that your scap repository is named $title/scap. +# +# To use this in conjunction with scap::target, make sure the $title here +# matches a scap::target's $title on your target hosts, or at least matches +# the $package_name provided to scap::target (which defaults to $title). +# +# NOTE: This define is compatible with trebuchet's deployment.yaml file. +# If trebuchet has already cloned a source repository in /srv/deployment, +# this clone will do nothing, as it only executes if .git/config +# doesn't already exist. +# +# == Parameters +# +# [*repository*] +# Repository name in gerrit. Default: $title +# +# [*scap_repository*] +# String or boolean. +# +# If you set this to a string, it will be assumed to be a repository name +# This scap repository will then be cloned into /srv/deployment/$title/scap. +# If this is set to true your scap_repository will be assumed to +# live at $title/scap in gerrit. +# +# You can use this keep your scap configs separate from your source +# repositories. +# +# Default: false. +# +# [*owner*] +# Owner of cloned repository, +# Default: trebuchet +# +# [*group*] +# Group owner of cloned repository. +# Default: wikidev +# +# == Usage +# +# # Clones the 'repo/without/external/scap' repsitory into +# # /srv/deployment/repo/without/external/scap. +# +# scap_source { 'repo/without/external/scap': } +# +# +# # Clones the 'eventlogging' repository into +# # /srv/deployment/eventlogging/eventbus and +# # clones the 'eventlogging/eventbus/scap' repository +# # into /srv/deployment/eventlogging/eventbus/scap +# +# scap_source { 'eventlogging/eventbus': +# repository => 'eventlogging', +# scap_repository => true, +# } +# +# +# # Clones the 'myproject/myrepo' repository into +# # /srv/deployment/myproject/myrepo, and +# # clones the custom scap repository at +# # 'my/custom/scap/repo' from gerrit into +# # /srv/deployment/myproject/myrepo/scap +# +# scap_source { 'myproject/myrepo': +# scap_repository => 'my/custom/scap/repo', +# } +# +Puppet::Type.newtype(:scap_source) do + @doc = "Puppet type to set up scap repositories on the scap master" + + ensurable do + desc <<-EOT +If the repository must be set up or not +EOT + newvalue(:present, :event => :scap_source_created) do + provider.create + end + + newvalue(:absent, :event => :scap_source_removed) do + provider.destroy + end + + defaultto :present + end + + newparam(:name, :namevar => true) do + desc "Name of the scap source" + end + + newparam(:owner) do + desc "Owner of the cloned repository. Defaults to 'trebuchet'" + + defaultto 'trebuchet' + end + + newparam(:group) do + desc "Group owner of the cloned repository. Defaults to 'wikidev'" + + defaultto 'wikidev' + end + + newparam(:repository) do + desc <<-EOT +Repository name in the VCS. Defaults to the resource name +EOT + defaultto do + @resource[:name] + end + end + + + newparam(:scap_repository) do + desc <<-EOT +String or boolean. + +If you set this to a string, it will be assumed to be a repository name +This scap repository will then be cloned into /srv/deployment/$title/scap. +If this is set to true your scap_repository will be assumed to +live at $title/scap in gerrit. + +You can use this keep your scap configs separate from your source +repositories. +Default: false. +EOT + defaultto false + munge do |value| + case value + when false, :false + false + when true, :true + File.join(@resource[:name], 'scap') + else + value + end + end + end +end diff --git a/modules/scap/manifests/server.pp b/modules/scap/manifests/server.pp index e163efc..655220c 100644 --- a/modules/scap/manifests/server.pp +++ b/modules/scap/manifests/server.pp @@ -3,7 +3,7 @@ # Configures dependencies for a scap3 deployment server. This includes # setting up ssh agent keys and repositories configured for deployment. # -# This class creates keyholder::agent and scap::source resources based on +# This class creates keyholder::agent and scap_source resources based on # the contents of the 'keyholder::agents' and 'scap::sources' hiera variables. # These would be class parameters instead of hiera lookups, if it were possible # to do a hiera hash merge using class parameters. Since hash merge doesn't @@ -29,14 +29,14 @@ # which is kept in a private location in the puppet modulepath. # # [*sources*] -# Hash of scap::source resource declarations to be passed to +# Hash of scap_source resource declarations to be passed to # the create_resources() function. Default: {} # # Each repository listed will be cloned via declaration of the -# scap::source define. You should use scap::target directly on your +# scap_source type. You should use scap::target directly on your # target hosts that are declared with $package_name matching the keys in # this hash. -# See scap::source for more information. +# See scap_source for more information. # # == Usage # @@ -62,6 +62,6 @@ # Create an instance of $keyholder_agents for each of the key specs. create_resources('keyholder::agent', $keyholder_agents) - # Create an instance of scap::source for each of the key specs in hiera. - create_resources('scap::source', $sources) + # Create an instance of scap_source for each of the key specs in hiera. + create_resources('scap_source', $sources) } -- To view, visit https://gerrit.wikimedia.org/r/308973 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: If8ed793759ebce183ef7ec8f36decfd287438d4d Gerrit-PatchSet: 7 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
