[WHIRR-548] Whirr can now reuse existing compute services. Misc osgi/karaf integration improvements.
Project: http://git-wip-us.apache.org/repos/asf/whirr/repo Commit: http://git-wip-us.apache.org/repos/asf/whirr/commit/b8b87cf9 Tree: http://git-wip-us.apache.org/repos/asf/whirr/tree/b8b87cf9 Diff: http://git-wip-us.apache.org/repos/asf/whirr/diff/b8b87cf9 Branch: refs/heads/branch-0.8 Commit: b8b87cf9babc94432b257e4d84574cc0f6b16c56 Parents: 27506ae Author: Ioannis Canellos <ioca...@apache.org> Authored: Mon Sep 10 14:44:23 2012 +0300 Committer: Ioannis Canellos <ioca...@apache.org> Committed: Wed Sep 19 09:42:11 2012 +0300 ---------------------------------------------------------------------- .../org/apache/whirr/ByonClusterController.java | 9 + .../main/java/org/apache/whirr/osgi/Activator.java | 38 +++++- .../apache/whirr/service/DynamicComputeCache.java | 99 +++++++++++++ platforms/karaf/commands/pom.xml | 1 - .../apache/whirr/karaf/command/LaunchCluster.java | 2 +- .../karaf/command/completers/RoleCompleter.java | 59 ++++++++ .../command/completers/TemplateCompleter.java | 60 ++++++++ .../karaf/command/support/WhirrCommandSupport.java | 115 +++++++++++++-- .../resources/OSGI-INF/blueprint/blueprint.xml | 86 +++++++++++ .../karaf/feature/src/main/resources/features.xml | 1 + .../ElasticSearchConfigurationBuilder.java | 2 +- .../service/hadoop/HadoopClusterActionHandler.java | 8 + .../service/hadoop/HadoopConfigurationBuilder.java | 8 +- .../hama/integration/HamaServiceController.java | 2 +- .../service/hbase/HBaseConfigurationBuilder.java | 4 +- .../hbase/HBaseMasterClusterActionHandler.java | 8 +- .../HBaseRegionServerClusterActionHandler.java | 6 + .../apache/whirr/service/hbase/osgi/Activator.java | 4 - .../service/yarn/YarnConfigurationBuilder.java | 2 +- 19 files changed, 481 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/core/src/main/java/org/apache/whirr/ByonClusterController.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/whirr/ByonClusterController.java b/core/src/main/java/org/apache/whirr/ByonClusterController.java index 538bafe..53b6c8a 100644 --- a/core/src/main/java/org/apache/whirr/ByonClusterController.java +++ b/core/src/main/java/org/apache/whirr/ByonClusterController.java @@ -30,6 +30,7 @@ import java.util.Set; import org.apache.whirr.Cluster.Instance; import org.apache.whirr.actions.ByonClusterAction; import org.apache.whirr.service.ClusterActionHandler; +import org.apache.whirr.state.ClusterStateStoreFactory; import org.jclouds.compute.ComputeService; import org.jclouds.compute.ComputeServiceContext; import org.jclouds.compute.RunScriptOnNodesException; @@ -50,6 +51,14 @@ import com.google.common.collect.Collections2; */ public class ByonClusterController extends ClusterController { + public ByonClusterController() { + super(); + } + + public ByonClusterController(Function<ClusterSpec, ComputeServiceContext> getCompute, ClusterStateStoreFactory stateStoreFactory) { + super(getCompute, stateStoreFactory); + } + @Override public String getName() { return "byon"; http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/core/src/main/java/org/apache/whirr/osgi/Activator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/whirr/osgi/Activator.java b/core/src/main/java/org/apache/whirr/osgi/Activator.java index 5a3c905..44818c0 100644 --- a/core/src/main/java/org/apache/whirr/osgi/Activator.java +++ b/core/src/main/java/org/apache/whirr/osgi/Activator.java @@ -24,6 +24,9 @@ import org.apache.whirr.ClusterControllerFactory; import org.apache.whirr.DynamicClusterControllerFactory; import org.apache.whirr.DynamicHandlerMapFactory; import org.apache.whirr.service.ClusterActionHandler; +import org.apache.whirr.service.DynamicComputeCache; +import org.apache.whirr.state.ClusterStateStoreFactory; +import org.jclouds.compute.ComputeService; import org.jclouds.scriptbuilder.functionloader.osgi.BundleFunctionLoader; import org.osgi.framework.BundleActivator; import org.osgi.framework.BundleContext; @@ -36,6 +39,11 @@ import java.util.Properties; public class Activator implements BundleActivator { + private ClusterStateStoreFactory clusterStateStoreFactory = new ClusterStateStoreFactory(); + + private DynamicComputeCache dynamicComputeCache = new DynamicComputeCache(); + private ServiceTracker computeServiceTracker; + private DynamicHandlerMapFactory handlerMapFactory = new DynamicHandlerMapFactory(); private ServiceRegistration handlerMapFactoryRegistration; private ServiceTracker handlerTracker; @@ -44,12 +52,13 @@ public class Activator implements BundleActivator { private ServiceRegistration clusterControllerFactoryRegistration; private ServiceTracker clusterControllerTracker; - private ClusterController defaultClusterController = new ClusterController(); + private ClusterController defaultClusterController = new ClusterController(dynamicComputeCache, clusterStateStoreFactory); private ServiceRegistration defaultClusterControllerRegistration; - private ClusterController byonClusterController = new ByonClusterController(); + private ClusterController byonClusterController = new ByonClusterController(dynamicComputeCache, clusterStateStoreFactory); private ServiceRegistration byonClusterControllerRegistration; + private BundleFunctionLoader functionLoader; /** @@ -73,7 +82,10 @@ public class Activator implements BundleActivator { functionLoader = new BundleFunctionLoader(context); functionLoader.start(); - //Register services + defaultClusterController.setHandlerMapFactory(handlerMapFactory); + byonClusterController.setHandlerMapFactory(handlerMapFactory); + + //Register services clusterControllerFactoryRegistration = context.registerService(ClusterControllerFactory.class.getName(), clusterControllerFactory, null); handlerMapFactoryRegistration = context.registerService(DynamicHandlerMapFactory.class.getName(), handlerMapFactory, null); @@ -96,12 +108,30 @@ public class Activator implements BundleActivator { clusterControllerTracker.open(); + computeServiceTracker = new ServiceTracker(context, ComputeService.class.getName(), null) { + + @Override + public Object addingService(ServiceReference reference) { + Object service = context.getService(reference); + dynamicComputeCache.bind((ComputeService) service); + return service; + } + + @Override + public void removedService(ServiceReference reference, Object service) { + dynamicComputeCache.unbind((ComputeService) service); + super.removedService(reference, service); + } + }; + + computeServiceTracker.open(); + handlerTracker = new ServiceTracker(context, ClusterActionHandler.class.getName(), null) { @Override public Object addingService(ServiceReference reference) { - Object service = super.addingService(reference); + Object service = context.getService(reference); handlerMapFactory.bind((ClusterActionHandler) service); return service; } http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/core/src/main/java/org/apache/whirr/service/DynamicComputeCache.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/whirr/service/DynamicComputeCache.java b/core/src/main/java/org/apache/whirr/service/DynamicComputeCache.java new file mode 100644 index 0000000..ec9381c --- /dev/null +++ b/core/src/main/java/org/apache/whirr/service/DynamicComputeCache.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.whirr.service; + +import com.google.common.base.Function; +import com.google.common.base.Objects; +import org.apache.whirr.ClusterSpec; +import org.jclouds.compute.ComputeService; +import org.jclouds.compute.ComputeServiceContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +/** + * A class for adding {@link ComputeServiceContext} on runtime. + */ +public class DynamicComputeCache implements Function<ClusterSpec, ComputeServiceContext> { + + private static final Logger LOG = LoggerFactory.getLogger(DynamicComputeCache.class); + + private final Map<Key, ComputeServiceContext> serviceContextMap = new HashMap<Key, ComputeServiceContext>(); + + @Override + public ComputeServiceContext apply(ClusterSpec arg0) { + return serviceContextMap.get(new Key(arg0)); + } + + public void bind(ComputeService computeService) { + if (computeService != null) { + serviceContextMap.put(new Key(computeService), computeService.getContext()); + } + } + + public void unbind(ComputeService computeService) { + if (computeService != null) { + serviceContextMap.remove(new Key(computeService)); + } + } + + /** + * Key class for the compute context cache + */ + private static class Key { + private String provider; + private final String key; + + public Key(ClusterSpec spec) { + provider = spec.getProvider(); + + key = Objects.toStringHelper("").omitNullValues() + .add("provider", provider).toString(); + } + + public Key(ComputeService computeService) { + provider = computeService.getContext().unwrap().getId(); + + key = Objects.toStringHelper("").omitNullValues() + .add("provider", provider).toString(); + } + + @Override + public boolean equals(Object that) { + if (that instanceof Key) { + return Objects.equal(this.key, ((Key)that).key); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hashCode(key); + } + + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("provider", provider) + .toString(); + } + } +} http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/platforms/karaf/commands/pom.xml ---------------------------------------------------------------------- diff --git a/platforms/karaf/commands/pom.xml b/platforms/karaf/commands/pom.xml index 70fdb6e..f50e8a1 100644 --- a/platforms/karaf/commands/pom.xml +++ b/platforms/karaf/commands/pom.xml @@ -38,7 +38,6 @@ org.apache.commons.configuration*;version="[1.6,2)", * </osgi.import> - <osgi.dynamic.import>javax.*,org.jclouds.*</osgi.dynamic.import> <osgi.export>org.apache.whirr.karaf.command*;version="${project.version}" </osgi.export> </properties> http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/LaunchCluster.java ---------------------------------------------------------------------- diff --git a/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/LaunchCluster.java b/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/LaunchCluster.java index e31b801..65fc542 100644 --- a/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/LaunchCluster.java +++ b/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/LaunchCluster.java @@ -28,7 +28,7 @@ public class LaunchCluster extends WhirrCommandSupport { @Override protected Object doExecute() throws Exception { - Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); + validateInput(); LaunchClusterCommand command = new LaunchClusterCommand(clusterControllerFactory); ClusterSpec clusterSpec = getClusterSpec(); if (clusterSpec != null) { http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/completers/RoleCompleter.java ---------------------------------------------------------------------- diff --git a/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/completers/RoleCompleter.java b/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/completers/RoleCompleter.java new file mode 100644 index 0000000..615c4b6 --- /dev/null +++ b/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/completers/RoleCompleter.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.whirr.karaf.command.completers; + +import org.apache.karaf.shell.console.Completer; +import org.apache.karaf.shell.console.completer.StringsCompleter; +import org.apache.whirr.service.ClusterActionHandler; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +public class RoleCompleter implements Completer { + + protected final StringsCompleter delegate = new StringsCompleter(); + private List<ClusterActionHandler> clusterActionHandlers; + + @Override + public int complete(String buffer, int cursor, List<String> candidates) { + delegate.getStrings().clear(); + for (String role : getRoles()) { + delegate.getStrings().add(role); + } + return delegate.complete(buffer, cursor, candidates); + } + + public Set<String> getRoles() { + Set<String> roles = new LinkedHashSet(); + if (clusterActionHandlers != null && !clusterActionHandlers.isEmpty()) { + for (ClusterActionHandler clusterActionHandler : clusterActionHandlers) { + roles.add(clusterActionHandler.getRole()); + } + } + return roles; + } + + public List<ClusterActionHandler> getClusterActionHandlers() { + return clusterActionHandlers; + } + + public void setClusterActionHandlers(List<ClusterActionHandler> clusterActionHandlers) { + this.clusterActionHandlers = clusterActionHandlers; + } +} http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/completers/TemplateCompleter.java ---------------------------------------------------------------------- diff --git a/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/completers/TemplateCompleter.java b/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/completers/TemplateCompleter.java new file mode 100644 index 0000000..c6eb746 --- /dev/null +++ b/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/completers/TemplateCompleter.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.whirr.karaf.command.completers; + +import org.apache.karaf.shell.console.Completer; + +import java.util.LinkedList; +import java.util.List; + +public class TemplateCompleter implements Completer { + + private RoleCompleter roleCompleter; + + @Override + public int complete(String buffer, int cursor, List<String> candidates) { + if (buffer == null || !(buffer.contains("[") || buffer.contains("+") || buffer.contains(",") )) { + return roleCompleter.complete(buffer, cursor, candidates); + } else { + int lastNumRoleDelimeter = buffer.lastIndexOf(",") + 1; + int lastRoleDelimeter = buffer.lastIndexOf("+") + 1; + int roleOpener = buffer.lastIndexOf("[") + 1; + + int pivot = Math.max(Math.max(lastNumRoleDelimeter, lastRoleDelimeter), roleOpener); + int result = roleCompleter.complete(buffer.substring(pivot), cursor, candidates); + List<String> updatedCandidates = new LinkedList<String>(); + for (String candidate : candidates) { + candidate = buffer.substring(0, pivot) + candidate; + updatedCandidates.add(candidate); + } + candidates.clear(); + for (String candidate : updatedCandidates) { + candidates.add(candidate); + } + return result; + } + } + + public RoleCompleter getRoleCompleter() { + return roleCompleter; + } + + public void setRoleCompleter(RoleCompleter roleCompleter) { + this.roleCompleter = roleCompleter; + } +} http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/support/WhirrCommandSupport.java ---------------------------------------------------------------------- diff --git a/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/support/WhirrCommandSupport.java b/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/support/WhirrCommandSupport.java index b1d8761..697e7f9 100644 --- a/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/support/WhirrCommandSupport.java +++ b/platforms/karaf/commands/src/main/java/org/apache/whirr/karaf/command/support/WhirrCommandSupport.java @@ -23,10 +23,14 @@ import org.apache.felix.gogo.commands.Option; import org.apache.karaf.shell.console.OsgiCommandSupport; import org.apache.whirr.ClusterControllerFactory; import org.apache.whirr.ClusterSpec; +import org.apache.whirr.InstanceTemplate; import org.jclouds.compute.ComputeService; +import org.jclouds.compute.domain.TemplateBuilderSpec; import org.osgi.service.cm.ConfigurationAdmin; +import java.util.ArrayList; import java.util.List; +import java.util.Map; public abstract class WhirrCommandSupport extends OsgiCommandSupport { @@ -36,6 +40,24 @@ public abstract class WhirrCommandSupport extends OsgiCommandSupport { @Option(required = false, name = "--pid", description = "The PID of the configuration") protected String pid; + @Option(required = false, name = "--provider", description = "The compute provider") + protected String provider; + + @Option(required = false, name = "--endpoint", description = "The compute endpoint") + protected String endpoint; + + @Option(required = false, name = "--templates", description = "The templates to use") + protected String templates; + + @Option(required = false, name = "--imageId", description = "The image id") + protected String imageId; + + @Option(required = false, name = "--locationId", description = "The location") + protected String locationId; + + @Option(required = false, name = "--hardwareId", description = "The hardware") + protected String hardwareId; + @Option(required = false, name = "--cluster-name", description = "The name of the cluster") protected String clusterName; @@ -46,6 +68,31 @@ public abstract class WhirrCommandSupport extends OsgiCommandSupport { protected ConfigurationAdmin configurationAdmin; protected List<ComputeService> computeServices; + + public void validateInput() throws Exception { + if (pid != null || fileName != null) { + return; + } else { + if (provider == null || getComputeService(provider) == null) { + throw new Exception("A proper configuration or a valid provider should be provided."); + } + if (templates == null) { + throw new Exception("A proper configuration or a valid template should be specified"); + } + } + } + + public ComputeService getComputeService(String provider) { + if (computeServices != null && !computeServices.isEmpty()) { + for (ComputeService computeService : computeServices) { + if (computeService.getContext().unwrap().getId().equals(provider)) { + return computeService; + } + } + } + return null; + } + /** * Returns the {@link ClusterSpec} * @@ -54,19 +101,66 @@ public abstract class WhirrCommandSupport extends OsgiCommandSupport { */ protected ClusterSpec getClusterSpec() throws Exception { ClusterSpec clusterSpec = null; - PropertiesConfiguration properties = getConfiguration(pid, fileName); - if (properties != null) { + if (pid != null || fileName != null) { + PropertiesConfiguration properties = getConfiguration(pid, fileName); clusterSpec = new ClusterSpec(properties); - if (privateKey != null) { - clusterSpec.setPrivateKey(privateKey); - } - if (clusterName != null) { - clusterSpec.setClusterName(clusterName); - } + } else { + clusterSpec = new ClusterSpec(); } + + + + if (provider != null) { + clusterSpec.setProvider(provider); + } + + if (endpoint != null) { + clusterSpec.setEndpoint(endpoint); + } + + if (templates != null) { + clusterSpec.setInstanceTemplates(getTemplate(templates, imageId, hardwareId)); + } + + + if (privateKey != null) { + clusterSpec.setPrivateKey(privateKey); + } + if (clusterName != null) { + clusterSpec.setClusterName(clusterName); + } + return clusterSpec; } + protected InstanceTemplate.Builder getTemplateBuilder(String imageId, String hardwareId) { + InstanceTemplate.Builder instanceBuilder = InstanceTemplate.builder(); + StringBuilder sb = new StringBuilder(); + String separator = ""; + if (imageId != null) { + sb.append(separator).append("imageId=").append(imageId); + separator = ","; + } + + if (hardwareId != null) { + sb.append(separator).append("hardwareId=").append(hardwareId); + } + instanceBuilder = instanceBuilder.template(TemplateBuilderSpec.parse(sb.toString())); + return instanceBuilder; + } + + protected List<InstanceTemplate> getTemplate(String templates, String imageId, String hardwareId) { + List<InstanceTemplate> templateList = new ArrayList<InstanceTemplate>(); + Map<String, String> roleMap = InstanceTemplate.parse(templates.replaceAll("\\["," ").replaceAll("\\]"," ").split(",")); + for (Map.Entry<String, String> entry : roleMap.entrySet()) { + InstanceTemplate.Builder builder = getTemplateBuilder(imageId, hardwareId); + Integer numberOfInstances = Integer.parseInt(entry.getValue()); + String roles = entry.getKey(); + templateList.add(builder.numberOfInstance(numberOfInstances).roles(roles.split("\\+")).build()); + } + return templateList; + } + /** * Retrieves the configuration from a pid or a file. * @@ -101,12 +195,7 @@ public abstract class WhirrCommandSupport extends OsgiCommandSupport { this.configurationAdmin = configurationAdmin; } - public List<ComputeService> getComputeServices() { - return computeServices; - } - public void setComputeServices(List<ComputeService> computeServices) { this.computeServices = computeServices; } - } http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/platforms/karaf/commands/src/main/resources/OSGI-INF/blueprint/blueprint.xml ---------------------------------------------------------------------- diff --git a/platforms/karaf/commands/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/platforms/karaf/commands/src/main/resources/OSGI-INF/blueprint/blueprint.xml new file mode 100644 index 0000000..4c1f0d0 --- /dev/null +++ b/platforms/karaf/commands/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -0,0 +1,86 @@ +<?xml version="1.0" encoding="ISO-8859-1"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> +<blueprint xmlns="http://www.osgi.org/xmlns/blueprint/v1.0.0";> + + <command-bundle xmlns="http://karaf.apache.org/xmlns/shell/v1.1.0";> + <command name="whirr/launch-cluster"> + <action class="org.apache.whirr.karaf.command.LaunchCluster"> + <property name="clusterControllerFactory" ref="clusterControllerFactory"/> + <property name="configurationAdmin" ref="configurationAdmin"/> + <property name="computeServices" ref="computeServices"/> + </action> + <optional-completers> + <entry key="--provider" value-ref="computeProviderCompleter"/> + <entry key="--imageId" value-ref="imageCompleter"/> + <entry key="--locationId" value-ref="locationCompleter"/> + <entry key="--hardwareId" value-ref="hardwareCompleter"/> + <entry key="--templates" value-ref="templateCompleter"/> + </optional-completers> + </command> + <command name="whirr/list-cluster"> + <action class="org.apache.whirr.karaf.command.ListCluster"> + <property name="clusterControllerFactory" ref="clusterControllerFactory"/> + <property name="configurationAdmin" ref="configurationAdmin"/> + <property name="computeServices" ref="computeServices"/> + </action> + </command> + <command name="whirr/destroy-cluster"> + <action class="org.apache.whirr.karaf.command.DestroyCluster"> + <property name="clusterControllerFactory" ref="clusterControllerFactory"/> + <property name="configurationAdmin" ref="configurationAdmin"/> + <property name="computeServices" ref="computeServices"/> + </action> + </command> + <command name="whirr/destroy-instance"> + <action class="org.apache.whirr.karaf.command.DestroyInstance"> + <property name="clusterControllerFactory" ref="clusterControllerFactory"/> + <property name="configurationAdmin" ref="configurationAdmin"/> + <property name="computeServices" ref="computeServices"/> + </action> + </command> + <command name="whirr/run-script"> + <action class="org.apache.whirr.karaf.command.RunScript"> + <property name="clusterControllerFactory" ref="clusterControllerFactory"/> + <property name="configurationAdmin" ref="configurationAdmin"/> + <property name="computeServices" ref="computeServices"/> + </action> + </command> + </command-bundle> + + <bean id="roleCompleter" class="org.apache.whirr.karaf.command.completers.RoleCompleter"> + <property name="clusterActionHandlers" ref="clusterActionHandlers"/> + </bean> + + <bean id="templateCompleter" class="org.apache.whirr.karaf.command.completers.TemplateCompleter"> + <property name="roleCompleter" ref="roleCompleter"/> + </bean> + + <reference id="imageCompleter" interface="org.apache.karaf.shell.console.Completer" filter="(completer-type=image)"/> + <reference id="locationCompleter" interface="org.apache.karaf.shell.console.Completer" filter="(completer-type=location)"/> + <reference id="hardwareCompleter" interface="org.apache.karaf.shell.console.Completer" filter="(completer-type=hardware)"/> + <reference id="computeProviderCompleter" interface="org.apache.karaf.shell.console.Completer" filter="(completer-type=compute-provider)"/> + + <reference id="clusterControllerFactory" interface="org.apache.whirr.ClusterControllerFactory"/> + <reference id="configurationAdmin" interface="org.osgi.service.cm.ConfigurationAdmin"/> + + <reference-list id="computeServices" interface="org.jclouds.compute.ComputeService" availability="optional"/> + <reference-list id="clusterActionHandlers" interface="org.apache.whirr.service.ClusterActionHandler" availability="optional"/> + +</blueprint> http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/platforms/karaf/feature/src/main/resources/features.xml ---------------------------------------------------------------------- diff --git a/platforms/karaf/feature/src/main/resources/features.xml b/platforms/karaf/feature/src/main/resources/features.xml index 75fada8..6d3016e 100644 --- a/platforms/karaf/feature/src/main/resources/features.xml +++ b/platforms/karaf/feature/src/main/resources/features.xml @@ -20,6 +20,7 @@ <repository>mvn:org.jclouds.karaf/jclouds-karaf/${jclouds.karaf.version}/xml/features</repository> <feature name="whirr" version="${project.version}" description="Apache Whirr Core" resolver="(obr)"> + <feature version="${jclouds.karaf.version}">jclouds-commands</feature> <feature version="${jclouds.karaf.version}">jclouds-aws-ec2</feature> <feature version="${jclouds.karaf.version}">jclouds-aws-s3</feature> <bundle dependency='true'>mvn:org.apache.geronimo.specs/geronimo-activation_1.1_spec/${activation.spec.version}</bundle> http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchConfigurationBuilder.java ---------------------------------------------------------------------- diff --git a/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchConfigurationBuilder.java b/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchConfigurationBuilder.java index 269d003..6dbcc45 100644 --- a/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchConfigurationBuilder.java +++ b/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchConfigurationBuilder.java @@ -66,7 +66,7 @@ public class ElasticSearchConfigurationBuilder { config.addConfiguration(spec.getConfiguration()); try { config.addConfiguration( - new PropertiesConfiguration("whirr-elasticsearch-default.properties")); + new PropertiesConfiguration(ElasticSearchConfigurationBuilder.class.getResource("/whirr-elasticsearch-default.properties"))); } catch (ConfigurationException e) { LOG.error("Configuration error", e); // this should never happen } http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopClusterActionHandler.java b/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopClusterActionHandler.java index 698216f..26c60ca 100644 --- a/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopClusterActionHandler.java +++ b/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopClusterActionHandler.java @@ -115,7 +115,13 @@ public abstract class HadoopClusterActionHandler extends ClusterActionHandlerSup private void createHadoopConfigFiles(ClusterActionEvent event, ClusterSpec clusterSpec, Cluster cluster) throws IOException { Map<String, String> deviceMappings = getDeviceMappings(event); + + //Velocity is assuming flat classloaders or TCCL to load templates. + //This doesn't work in OSGi unless we set the TCCL to the bundle classloader before invocation + ClassLoader oldTccl = Thread.currentThread().getContextClassLoader(); try { + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); + event.getStatementBuilder().addStatements( buildCommon("/tmp/core-site.xml", clusterSpec, cluster), buildHdfs("/tmp/hdfs-site.xml", clusterSpec, cluster, deviceMappings.keySet()), @@ -126,6 +132,8 @@ public abstract class HadoopClusterActionHandler extends ClusterActionHandlerSup } catch (ConfigurationException e) { throw new IOException(e); + } finally { + Thread.currentThread().setContextClassLoader(oldTccl); } String devMappings = VolumeManager.asString(deviceMappings); addStatement(event, call("prepare_all_disks", "'" + devMappings + "'")); http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java ---------------------------------------------------------------------- diff --git a/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java b/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java index 2916149..b61314f 100644 --- a/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java +++ b/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java @@ -63,28 +63,28 @@ public class HadoopConfigurationBuilder { public static Statement buildCommon(String path, ClusterSpec clusterSpec, Cluster cluster) throws ConfigurationException, IOException { Configuration config = buildCommonConfiguration(clusterSpec, cluster, - new PropertiesConfiguration(WHIRR_HADOOP_DEFAULT_PROPERTIES)); + new PropertiesConfiguration(HadoopConfigurationBuilder.class.getResource("/" + WHIRR_HADOOP_DEFAULT_PROPERTIES))); return HadoopConfigurationConverter.asCreateXmlConfigurationFileStatement(path, config); } public static Statement buildHdfs(String path, ClusterSpec clusterSpec, Cluster cluster, Set<String> dataDirectories) throws ConfigurationException, IOException { Configuration config = buildHdfsConfiguration(clusterSpec, cluster, - new PropertiesConfiguration(WHIRR_HADOOP_DEFAULT_PROPERTIES), dataDirectories); + new PropertiesConfiguration(HadoopConfigurationBuilder.class.getResource("/" + WHIRR_HADOOP_DEFAULT_PROPERTIES)), dataDirectories); return HadoopConfigurationConverter.asCreateXmlConfigurationFileStatement(path, config); } public static Statement buildMapReduce(String path, ClusterSpec clusterSpec, Cluster cluster, Set<String> dataDirectories) throws ConfigurationException, IOException { Configuration config = buildMapReduceConfiguration(clusterSpec, cluster, - new PropertiesConfiguration(WHIRR_HADOOP_DEFAULT_PROPERTIES), dataDirectories); + new PropertiesConfiguration(HadoopConfigurationBuilder.class.getResource("/" + WHIRR_HADOOP_DEFAULT_PROPERTIES)), dataDirectories); return HadoopConfigurationConverter.asCreateXmlConfigurationFileStatement(path, config); } public static Statement buildHadoopEnv(String path, ClusterSpec clusterSpec, Cluster cluster) throws ConfigurationException, IOException { Configuration config = buildHadoopEnvConfiguration(clusterSpec, cluster, - new PropertiesConfiguration(WHIRR_HADOOP_DEFAULT_PROPERTIES)); + new PropertiesConfiguration(HadoopConfigurationBuilder.class.getResource("/" + WHIRR_HADOOP_DEFAULT_PROPERTIES))); return HadoopConfigurationConverter.asCreateEnvironmentVariablesFileStatement(path, config); } http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceController.java ---------------------------------------------------------------------- diff --git a/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceController.java b/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceController.java index fcb2d85..ab96895 100644 --- a/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceController.java +++ b/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceController.java @@ -69,7 +69,7 @@ public class HamaServiceController { if (System.getProperty("config") != null) { config.addConfiguration(new PropertiesConfiguration(System.getProperty("config"))); } - config.addConfiguration(new PropertiesConfiguration("whirr-hama-test.properties")); + config.addConfiguration(new PropertiesConfiguration(HamaServiceController.class.getResource("/whirr-hama-test.properties"))); clusterSpec = ClusterSpec.withTemporaryKeys(config); controller = new ClusterController(); http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseConfigurationBuilder.java ---------------------------------------------------------------------- diff --git a/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseConfigurationBuilder.java b/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseConfigurationBuilder.java index 32d7adc..12553a5 100644 --- a/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseConfigurationBuilder.java +++ b/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseConfigurationBuilder.java @@ -36,7 +36,7 @@ public class HBaseConfigurationBuilder { public static Statement buildHBaseSite(String path, ClusterSpec clusterSpec, Cluster cluster) throws ConfigurationException, IOException { Configuration config = buildHBaseSiteConfiguration(clusterSpec, cluster, - new PropertiesConfiguration(HBaseConstants.FILE_HBASE_DEFAULT_PROPERTIES)); + new PropertiesConfiguration(HBaseConfigurationBuilder.class.getResource("/" + HBaseConstants.FILE_HBASE_DEFAULT_PROPERTIES))); return HadoopConfigurationConverter.asCreateXmlConfigurationFileStatement(path, config); } @@ -57,7 +57,7 @@ public class HBaseConfigurationBuilder { public static Statement buildHBaseEnv(String path, ClusterSpec clusterSpec, Cluster cluster) throws ConfigurationException, IOException { Configuration config = buildHBaseEnvConfiguration(clusterSpec, cluster, - new PropertiesConfiguration(HBaseConstants.FILE_HBASE_DEFAULT_PROPERTIES)); + new PropertiesConfiguration(HBaseConfigurationBuilder.class.getResource("/" + HBaseConstants.FILE_HBASE_DEFAULT_PROPERTIES))); return HadoopConfigurationConverter.asCreateEnvironmentVariablesFileStatement(path, config); } http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java b/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java index 757a356..d9f7ed0 100644 --- a/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java +++ b/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java @@ -95,7 +95,11 @@ public class HBaseMasterClusterActionHandler extends HBaseClusterActionHandler { .ports(MASTER_WEB_UI_PORT, MASTER_PORT) ); + //Velocity is assuming flat classloaders or TCCL to load templates. + //This doesn't work in OSGi unless we set the TCCL to the bundle classloader before invocation + ClassLoader oldTccl = Thread.currentThread().getContextClassLoader(); try { + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); event.getStatementBuilder().addStatements( buildHBaseSite("/tmp/hbase-site.xml", clusterSpec, cluster), buildHBaseEnv("/tmp/hbase-env.sh", clusterSpec, cluster), @@ -104,7 +108,9 @@ public class HBaseMasterClusterActionHandler extends HBaseClusterActionHandler { } catch (ConfigurationException e) { throw new IOException(e); - } + } finally { + Thread.currentThread().setContextClassLoader(oldTccl); + } String master = masterPublicAddress.getHostName(); String quorum = ZooKeeperCluster.getHosts(cluster); http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java ---------------------------------------------------------------------- diff --git a/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java b/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java index a3873df..a6ea33e 100644 --- a/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java +++ b/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java @@ -86,7 +86,11 @@ public class HBaseRegionServerClusterActionHandler extends HBaseClusterActionHan .ports(REGIONSERVER_WEB_UI_PORT, REGIONSERVER_PORT) ); + //Velocity is assuming flat classloaders or TCCL to load templates. + //This doesn't work in OSGi unless we set the TCCL to the bundle classloader before invocation + ClassLoader oldTccl = Thread.currentThread().getContextClassLoader(); try { + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); event.getStatementBuilder().addStatements( buildHBaseSite("/tmp/hbase-site.xml", clusterSpec, cluster), buildHBaseEnv("/tmp/hbase-env.sh", clusterSpec, cluster), @@ -94,6 +98,8 @@ public class HBaseRegionServerClusterActionHandler extends HBaseClusterActionHan ); } catch (ConfigurationException e) { throw new IOException(e); + } finally { + Thread.currentThread().setContextClassLoader(oldTccl); } String master = masterPublicAddress.getHostName(); http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/services/hbase/src/main/java/org/apache/whirr/service/hbase/osgi/Activator.java ---------------------------------------------------------------------- diff --git a/services/hbase/src/main/java/org/apache/whirr/service/hbase/osgi/Activator.java b/services/hbase/src/main/java/org/apache/whirr/service/hbase/osgi/Activator.java index b499929..0c71708 100644 --- a/services/hbase/src/main/java/org/apache/whirr/service/hbase/osgi/Activator.java +++ b/services/hbase/src/main/java/org/apache/whirr/service/hbase/osgi/Activator.java @@ -113,7 +113,6 @@ public class Activator implements BundleActivator { if (masterRegistration != null) { masterRegistration.unregister(); } - if (regionServerRegistration != null) { regionServerRegistration.unregister(); } @@ -127,9 +126,6 @@ public class Activator implements BundleActivator { if (thriftServerRegistration != null) { thriftServerRegistration.unregister(); } - if (masterRegistration != null) { - masterRegistration.unregister(); - } if (functionLoader != null) { functionLoader.stop(); } http://git-wip-us.apache.org/repos/asf/whirr/blob/b8b87cf9/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnConfigurationBuilder.java ---------------------------------------------------------------------- diff --git a/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnConfigurationBuilder.java b/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnConfigurationBuilder.java index 355f124..69620a3 100644 --- a/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnConfigurationBuilder.java +++ b/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnConfigurationBuilder.java @@ -53,7 +53,7 @@ public class YarnConfigurationBuilder { public static Statement build(String path, ClusterSpec clusterSpec, Cluster cluster, String role) throws ConfigurationException, IOException { Configuration config = buildConfiguration(clusterSpec, cluster, role, - new PropertiesConfiguration(WHIRR_YARN_DEFAULT_PROPERTIES)); + new PropertiesConfiguration(YarnConfigurationBuilder.class.getResource("/" + WHIRR_YARN_DEFAULT_PROPERTIES))); return HadoopConfigurationConverter.asCreateXmlConfigurationFileStatement(path, config); }
