Updated Branches: refs/heads/trunk d5689610e -> 32481db44
WHIRR-681. Enhance puppet service with an ability to export cluster topology to the puppet code. Contributed by Roman Shaposhnik. Project: http://git-wip-us.apache.org/repos/asf/whirr/repo Commit: http://git-wip-us.apache.org/repos/asf/whirr/commit/32481db4 Tree: http://git-wip-us.apache.org/repos/asf/whirr/tree/32481db4 Diff: http://git-wip-us.apache.org/repos/asf/whirr/diff/32481db4 Branch: refs/heads/trunk Commit: 32481db44d8e5334423ac6285d33ef3608f9017d Parents: d568961 Author: Andrew Bayer <andrew.ba...@gmail.com> Authored: Mon Mar 11 11:09:01 2013 -0700 Committer: Andrew Bayer <andrew.ba...@gmail.com> Committed: Mon Mar 11 11:09:01 2013 -0700 ---------------------------------------------------------------------- CHANGES.txt | 3 + .../service/puppet/PuppetClusterActionHandler.java | 2 +- .../whirr/service/puppet/PuppetConstants.java | 1 + .../statements/CreateSitePpAndApplyRoles.java | 35 ++++++++++++++- .../src/main/resources/functions/install_puppet.sh | 2 +- .../statements/CreateSitePpAndApplyRolesTest.java | 13 +++++- .../src/test/resources/nginx-with-attribs.txt | 6 +++ 7 files changed, 58 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/whirr/blob/32481db4/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 1a57f5a..a465421 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,9 @@ Release 0.8.2 (unreleased changes) IMPROVEMENTS + WHIRR-681. Enhance puppet service with an ability to export + cluster topology to the puppet code. (Roman Shaposhnik via abayer) + WHIRR-661. Upgrade to Hadoop 1.0.3. (Steve Loughran via abayer) WHIRR-634. Update private IP host file entry when required by http://git-wip-us.apache.org/repos/asf/whirr/blob/32481db4/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java b/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java index f10279f..6b5170f 100644 --- a/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java +++ b/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java @@ -105,7 +105,7 @@ public class PuppetClusterActionHandler extends PuppetInstallClusterActionHandle if (isLastPuppetRoleIn(event.getInstanceTemplate().getRoles()).apply(getRole())) { Configuration config = event.getClusterSpec().getConfiguration(); Iterable<String> roles = RolesManagedByPuppet.INSTANCE.apply(event.getInstanceTemplate().getRoles()); - addStatement(event, new CreateSitePpAndApplyRoles(roles, config)); + addStatement(event, new CreateSitePpAndApplyRoles(roles, event.getCluster().getInstances(), config)); } } http://git-wip-us.apache.org/repos/asf/whirr/blob/32481db4/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetConstants.java ---------------------------------------------------------------------- diff --git a/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetConstants.java b/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetConstants.java index c6ccdaa..3aca26c 100644 --- a/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetConstants.java +++ b/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetConstants.java @@ -30,6 +30,7 @@ public class PuppetConstants { public static final String MODULES_DIR = "/etc/puppet/modules/"; public static final String SITE_PP_FILE_LOCATION = "/etc/puppet/manifests/site.pp"; + public static final String CONF_PP_FILE_LOCATION = "/etc/puppet/manifests/extdata/common.csv"; public static final Pattern MODULE_KEY_PATTERN = Pattern.compile("^" + PUPPET + "\\.([^.]+)\\." + MODULE_SOURCE_SUBKEY + "$"); http://git-wip-us.apache.org/repos/asf/whirr/blob/32481db4/services/puppet/src/main/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRoles.java ---------------------------------------------------------------------- diff --git a/services/puppet/src/main/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRoles.java b/services/puppet/src/main/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRoles.java index 2a15a28..6b65af1 100644 --- a/services/puppet/src/main/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRoles.java +++ b/services/puppet/src/main/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRoles.java @@ -20,15 +20,23 @@ package org.apache.whirr.service.puppet.statements; import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.whirr.service.puppet.PuppetConstants.PUPPET; import static org.apache.whirr.service.puppet.PuppetConstants.SITE_PP_FILE_LOCATION; +import static org.apache.whirr.service.puppet.PuppetConstants.CONF_PP_FILE_LOCATION; import static org.jclouds.scriptbuilder.domain.Statements.appendFile; import static org.jclouds.scriptbuilder.domain.Statements.exec; import java.util.Collection; import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import com.google.common.base.Joiner; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.whirr.Cluster; import org.apache.whirr.service.puppet.Manifest; import org.jclouds.scriptbuilder.domain.OsFamily; import org.jclouds.scriptbuilder.domain.Statement; @@ -44,10 +52,12 @@ import com.google.common.collect.ImmutableList.Builder; public class CreateSitePpAndApplyRoles implements Statement { private Iterable<String> roles; + private Iterable<Cluster.Instance> instances; private Configuration config; - public CreateSitePpAndApplyRoles(Iterable<String> roles, Configuration config) { + public CreateSitePpAndApplyRoles(Iterable<String> roles, Iterable<Cluster.Instance> instances, Configuration config) { this.roles = checkNotNull(roles, "roles"); + this.instances = checkNotNull(instances, "instances"); this.config = checkNotNull(config, "config"); } @@ -64,8 +74,30 @@ public class CreateSitePpAndApplyRoles implements Statement { Builder<Statement> statements = ImmutableList.<Statement> builder(); statements.add(Statements.rm(SITE_PP_FILE_LOCATION)); + statements.add(Statements.rm(CONF_PP_FILE_LOCATION)); Builder<String> sitePp = ImmutableList.<String> builder(); + Map<String, Set<String>> puppetRoles = Maps.newHashMap(); + for (Cluster.Instance instance : instances) { + for (String role : instance.getRoles()) { + int firstColon = role.indexOf(':'); + if (firstColon != -1 && role.substring(0, firstColon).equals(PUPPET)) { + String puppetClass = role.substring(firstColon + 1); + if (!puppetRoles.containsKey(puppetClass)) { + puppetRoles.put(puppetClass, Sets.<String>newHashSet()); + } + puppetRoles.get(puppetClass).add(instance.getPrivateIp()); + } + } + } + + Builder<String> confPp = ImmutableList.<String> builder(); + for (String puppetClass : puppetRoles.keySet()) { + confPp.add(puppetClass + "," + Joiner.on(',').join(puppetRoles.get(puppetClass))); + } + + sitePp.add("$extlookup_datadir='/etc/puppet/manifests/extdata'"); + sitePp.add("$extlookup_precedence = ['common']"); sitePp.add("node default {"); for (String role : roles) { String manifestAttribPrefix = role.replaceAll(":+", "."); @@ -79,6 +111,7 @@ public class CreateSitePpAndApplyRoles implements Statement { } sitePp.add("}"); + statements.add(appendFile(CONF_PP_FILE_LOCATION, confPp.build())); statements.add(appendFile(SITE_PP_FILE_LOCATION, sitePp.build())); statements.add(exec("puppet apply " + SITE_PP_FILE_LOCATION)); http://git-wip-us.apache.org/repos/asf/whirr/blob/32481db4/services/puppet/src/main/resources/functions/install_puppet.sh ---------------------------------------------------------------------- diff --git a/services/puppet/src/main/resources/functions/install_puppet.sh b/services/puppet/src/main/resources/functions/install_puppet.sh index 5c18161..e01a152 100644 --- a/services/puppet/src/main/resources/functions/install_puppet.sh +++ b/services/puppet/src/main/resources/functions/install_puppet.sh @@ -27,7 +27,7 @@ function install_puppet_forced() { # Setup the default module/fact path locations so we can populate them # elsewhere mkdir -p /etc/puppet/modules - mkdir -p /etc/puppet/manifests + mkdir -p /etc/puppet/manifests/extdata mkdir -p /usr/share/puppet/modules mkdir -p /var/lib/puppet/lib/facter http://git-wip-us.apache.org/repos/asf/whirr/blob/32481db4/services/puppet/src/test/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRolesTest.java ---------------------------------------------------------------------- diff --git a/services/puppet/src/test/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRolesTest.java b/services/puppet/src/test/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRolesTest.java index 3ff02b0..e404114 100644 --- a/services/puppet/src/test/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRolesTest.java +++ b/services/puppet/src/test/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRolesTest.java @@ -22,8 +22,11 @@ import static junit.framework.Assert.assertEquals; import java.io.IOException; +import com.google.common.collect.Sets; import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.whirr.Cluster; +import org.jclouds.domain.Credentials; import org.jclouds.scriptbuilder.domain.OsFamily; import org.junit.Test; @@ -40,7 +43,15 @@ public class CreateSitePpAndApplyRolesTest { conf.setProperty("puppet.nginx.module", "git://github.com/puppetlabs/puppetlabs-nginx.git"); conf.setProperty("nginx.server.hostname", "foohost"); - CreateSitePpAndApplyRoles nginx = new CreateSitePpAndApplyRoles(ImmutableSet.of("nginx::server"), conf); + CreateSitePpAndApplyRoles nginx = new CreateSitePpAndApplyRoles(ImmutableSet.of("nginx::server"), + ImmutableSet.of(new Cluster.Instance( + new Credentials("dummy", "dummy"), + Sets.newHashSet("puppet:nginx::server"), + "127.0.0.1", + "127.0.0.1", + "id-1", + null)), + conf); assertEquals(CharStreams.toString(Resources.newReaderSupplier(Resources.getResource("nginx-with-attribs.txt"), Charsets.UTF_8)), nginx.render(OsFamily.UNIX)); http://git-wip-us.apache.org/repos/asf/whirr/blob/32481db4/services/puppet/src/test/resources/nginx-with-attribs.txt ---------------------------------------------------------------------- diff --git a/services/puppet/src/test/resources/nginx-with-attribs.txt b/services/puppet/src/test/resources/nginx-with-attribs.txt index ed53c47..0bd63da 100644 --- a/services/puppet/src/test/resources/nginx-with-attribs.txt +++ b/services/puppet/src/test/resources/nginx-with-attribs.txt @@ -1,5 +1,11 @@ rm /etc/puppet/manifests/site.pp +rm /etc/puppet/manifests/extdata/common.csv +cat >> /etc/puppet/manifests/extdata/common.csv <<-'END_OF_JCLOUDS_FILE' + nginx::server,127.0.0.1 +END_OF_JCLOUDS_FILE cat >> /etc/puppet/manifests/site.pp <<-'END_OF_JCLOUDS_FILE' + $extlookup_datadir='/etc/puppet/manifests/extdata' + $extlookup_precedence = ['common'] node default { class { 'nginx::server': hostname => foohost,
