Thcipriani has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/365416 )
Change subject: CI/integration: Create profile for docker setup ...................................................................... CI/integration: Create profile for docker setup Change-Id: I6c6583790a981d20bdb727e3a3722a8fc419572a --- A modules/contint/manifests/git_cache.pp M modules/git/manifests/clone.pp A modules/profile/manifests/ci/docker.pp M modules/role/manifests/ci/slave/labs/docker.pp 4 files changed, 120 insertions(+), 45 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/16/365416/1 diff --git a/modules/contint/manifests/git_cache.pp b/modules/contint/manifests/git_cache.pp new file mode 100644 index 0000000..7af23f5 --- /dev/null +++ b/modules/contint/manifests/git_cache.pp @@ -0,0 +1,35 @@ +# == Define contint::git_cache +# +# Creates a bare repo in /srv/git so that the repo can be used by CI as a +# --reference for git clone operations. +# +# === Parameters +# +# [*repo*] repo that we need to cache on the docker hosts +# + +define contint::git_cache( + $repo = $title, +) { + $repo_path = "/srv/git/${repo}.git" + + exec { "mkdir ${repo_path}": + command => "/bin/mkdir -p ${repo_path}", + creates => $repo_path, + before => Git::Clone[$repo], + } + + # This is here to prevent the git::clone module from attempting to create + # multi-level repos directory before the parent directories are created by + # the exec in this definition + file { $repo_path: + ensure => 'directory', + require => Exec["mkdir ${repo_path}"], + } + + git::clone{ $repo: + bare => true, + directory => $repo_path, + shared => true, + } +} diff --git a/modules/git/manifests/clone.pp b/modules/git/manifests/clone.pp index ea310cf..38d616d 100644 --- a/modules/git/manifests/clone.pp +++ b/modules/git/manifests/clone.pp @@ -12,6 +12,7 @@ # checked out from gerrit using a default gerrit url. # If you set this, please specify the full repository url. # $+branch+:: Branch you would like to check out. +# $+bare+:: Create a bare repo # $+ensure+:: _absent_, _present_, or _latest_. Defaults to _present_. # - _present_ (default) will just clone once. # - _latest_ will execute a git pull if there are any changes. @@ -19,7 +20,8 @@ # $+owner+:: Owner of $directory, default: _root_. git commands will be run # by this user. # $+group+:: Group owner of $directory, default: 'root' -# $+recurse_submodules:: If true, git +# $+recurse_submodules:: If true, append --recurse-submodules to the clone +# command # $+shared+:: Enable git's core.sharedRepository=group setting for sharing the # repository between serveral users, default: false # $+umask+:: umask value that git operations should run under, @@ -49,6 +51,7 @@ define git::clone( $directory, $origin=undef, + $bare=false, $branch='', $ssh='', $ensure='present', @@ -135,18 +138,26 @@ $shared_arg = '' } + if $bare { + $bare_arg = '--bare ' + $creates = "${directory}/HEAD" + } else { + $bare_arg = '' + $creates = "${directory}/.git/config" + } + $git = '/usr/bin/git' # set PATH for following execs Exec { path => '/usr/bin:/bin' } # clone the repository exec { "git_clone_${title}": - command => "${git} ${shared_arg} clone ${recurse_submodules_arg}${brancharg}${remote}${deptharg} ${directory}", + command => "${git} ${shared_arg} clone ${recurse_submodules_arg}${brancharg}${bare_arg}${remote}${deptharg} ${directory}", provider => shell, logoutput => on_failure, cwd => '/tmp', environment => $env, - creates => "${directory}/.git/config", + creates => "${creates}", user => $owner, group => $group, umask => $git_umask, @@ -170,37 +181,50 @@ '' => 'remotes/origin/HEAD', default => "remotes/origin/${branch}", } - exec { "git_pull_${title}": - cwd => $directory, - command => "${git} ${shared_arg} pull ${recurse_submodules_arg}--quiet${deptharg}", - provider => shell, - logoutput => on_failure, - # git diff --quiet will exit 1 (return false) - # if there are differences - unless => "${git} fetch && ${git} diff --quiet ${remote_to_check}", - user => $owner, - group => $group, - umask => $git_umask, - require => Exec["git_clone_${title}"], - } - # If we want submodules up to date, then we need - # to run git submodule update --init after - # git pull is run. - if $recurse_submodules { - exec { "git_submodule_update_${title}": - command => "${git} ${shared_arg} submodule update --init", - provider => shell, - cwd => $directory, - environment => $env, - refreshonly => true, - user => $owner, - group => $group, - umask => $git_umask, - subscribe => Exec["git_pull_${title}"], + + if $bare { + exec { "git_fetch_${title}": + cwd => $directory, + command => "${git} ${shared_arg} fetch --quiet${deptharg}", + provider => shell, + logoutput => on_failure, + user => $owner, + group => $group, + umask => $git_umask, + require => Exec["git_clone_${title}"], + } + } else { + exec { "git_pull_${title}": + cwd => $directory, + command => "${git} ${shared_arg} pull ${recurse_submodules_arg}--quiet${deptharg}", + provider => shell, + logoutput => on_failure, + # git diff --quiet will exit 1 (return false) + # if there are differences + unless => "${git} fetch && ${git} diff --quiet ${remote_to_check}", + user => $owner, + group => $group, + umask => $git_umask, + require => Exec["git_clone_${title}"], + } + # If we want submodules up to date, then we need + # to run git submodule update --init after + # git pull is run. + if $recurse_submodules { + exec { "git_submodule_update_${title}": + command => "${git} ${shared_arg} submodule update --init", + provider => shell, + cwd => $directory, + environment => $env, + refreshonly => true, + user => $owner, + group => $group, + umask => $git_umask, + subscribe => Exec["git_pull_${title}"], + } } } } - } } } diff --git a/modules/profile/manifests/ci/docker.pp b/modules/profile/manifests/ci/docker.pp new file mode 100644 index 0000000..8f83b42 --- /dev/null +++ b/modules/profile/manifests/ci/docker.pp @@ -0,0 +1,28 @@ +# == Class profile::ci::docker +# +# Configures a host to be a docker-backed Jenkins agent +# +# === Parameters +# +# [*repos*] list of repos that we need to cache on the docker hosts +# + +class profile::ci::docker ( + $repos = hiera('profile::ci::docker::repos') +) { + include ::docker + include phabricator::arcanist + include ::zuul + + class { 'contint::worker_localhost': + owner => 'jenkins-deploy', + } + + # Ensure jenkins-deploy membership in the docker group + exec { 'jenkins-deploy docker membership': + unless => '/usr/bin/id -Gn jenkins-deploy | /bin/grep -q "\bdocker\b"', + command => '/usr/sbin/usermod -aG docker jenkins-deploy', + } + + create_resources(contint::git_cache, $repos) +} diff --git a/modules/role/manifests/ci/slave/labs/docker.pp b/modules/role/manifests/ci/slave/labs/docker.pp index 9a53630..0d59655 100644 --- a/modules/role/manifests/ci/slave/labs/docker.pp +++ b/modules/role/manifests/ci/slave/labs/docker.pp @@ -5,21 +5,9 @@ class role::ci::slave::labs::docker { requires_realm('labs') - include role::ci::slave::labs::common - include ::docker - include phabricator::arcanist - include ::zuul - system::role { 'role::ci::slave::labs::docker': description => 'CI Jenkins slave using Docker on labs' } - class { 'contint::worker_localhost': - owner => 'jenkins-deploy', - } - - # Ensure jenkins-deploy membership in the docker group - exec { 'jenkins-deploy docker membership': - unless => '/usr/bin/id -Gn jenkins-deploy | /bin/grep -q "\bdocker\b"', - command => '/usr/sbin/usermod -aG docker jenkins-deploy', - } + include role::ci::slave::labs::common + include profile::ci::docker } -- To view, visit https://gerrit.wikimedia.org/r/365416 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I6c6583790a981d20bdb727e3a3722a8fc419572a Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Thcipriani <tcipri...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits