Repository: gora Updated Branches: refs/heads/master 0c5eacf4e -> b712d9f9d
GORA-74 Implement Rackspace Cloud Orchestration in GoraCI Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/3c5bd0ee Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/3c5bd0ee Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/3c5bd0ee Branch: refs/heads/master Commit: 3c5bd0ee53ab5d3651980d39c5a5b20002ababc9 Parents: a72c5d3 Author: Lewis John McGibbney <[email protected]> Authored: Wed Sep 17 17:46:25 2014 -0400 Committer: Lewis John McGibbney <[email protected]> Committed: Wed Sep 17 17:46:25 2014 -0400 ---------------------------------------------------------------------- bin/gora | 5 + .../org/apache/gora/store/DataStoreFactory.java | 3 - .../libext/gora-core-tests-0.5-SNAPSHOT.jar | Bin 0 -> 128135 bytes gora-goraci/pom.xml | 24 ++- .../rackspace/RackspaceOrchestration.java | 151 +++++++++++++++++++ .../gora/goraci/rackspace/package-info.java | 22 +++ gora-goraci/src/main/resources/gora.properties | 55 +++++-- pom.xml | 2 +- 8 files changed, 244 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/3c5bd0ee/bin/gora ---------------------------------------------------------------------- diff --git a/bin/gora b/bin/gora index d660099..7126972 100755 --- a/bin/gora +++ b/bin/gora @@ -46,6 +46,7 @@ if [ $# = 0 ]; then echo " goracompiler Run Compiler" echo " specificcompiler Run Avro Specific Compiler" echo " dynamocompiler Run Gora DynamoDB Compiler" + echo " goracisetup Run the GoraCI Rackspace orchestration setup" echo " logmanager Run the tutorial log manager" echo " loganalytics Run the tutorial log analytics" echo " junit Run the given JUnit test" @@ -121,6 +122,10 @@ elif [ "$COMMAND" = "dynamocompiler" ] ; then MODULE=gora-dynamodb CLASSPATH=$CLASSPATH:$GORA_HOME/$MODULE/target/classes/ CLASS=org.apache.gora.dynamodb.compiler.GoraDynamoDBCompiler +elif [ "$COMMAND" = "goracisetup" ] ; then + MODULE=gora-goraci + CLASSPATH=$CLASSPATH:$GORA_HOME/$MODULE/target/classes/ + CLASS=org.apache.gora.goraci.rackspace.RackspaceOrchestration elif [ "$COMMAND" = "logmanager" ] ; then MODULE=gora-tutorial CLASSPATH=$CLASSPATH:$GORA_HOME/$MODULE/target/classes/ http://git-wip-us.apache.org/repos/asf/gora/blob/3c5bd0ee/gora-core/src/main/java/org/apache/gora/store/DataStoreFactory.java ---------------------------------------------------------------------- diff --git a/gora-core/src/main/java/org/apache/gora/store/DataStoreFactory.java b/gora-core/src/main/java/org/apache/gora/store/DataStoreFactory.java index 4408e80..b1d9c75 100644 --- a/gora-core/src/main/java/org/apache/gora/store/DataStoreFactory.java +++ b/gora-core/src/main/java/org/apache/gora/store/DataStoreFactory.java @@ -20,9 +20,6 @@ package org.apache.gora.store; import java.io.IOException; import java.io.InputStream; import java.util.Properties; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.gora.persistency.Persistent; http://git-wip-us.apache.org/repos/asf/gora/blob/3c5bd0ee/gora-goraci/libext/gora-core-tests-0.5-SNAPSHOT.jar ---------------------------------------------------------------------- diff --git a/gora-goraci/libext/gora-core-tests-0.5-SNAPSHOT.jar b/gora-goraci/libext/gora-core-tests-0.5-SNAPSHOT.jar new file mode 100755 index 0000000..48f0bb5 Binary files /dev/null and b/gora-goraci/libext/gora-core-tests-0.5-SNAPSHOT.jar differ http://git-wip-us.apache.org/repos/asf/gora/blob/3c5bd0ee/gora-goraci/pom.xml ---------------------------------------------------------------------- diff --git a/gora-goraci/pom.xml b/gora-goraci/pom.xml index d3d66c2..7011437 100644 --- a/gora-goraci/pom.xml +++ b/gora-goraci/pom.xml @@ -17,7 +17,8 @@ specific language governing permissions and limitations under the License. --> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> @@ -31,9 +32,9 @@ <name>Apache Gora :: GoraCI</name> <url>http://gora.apache.org</url> <description>The GoraCI test runs many ingest clients that continually create - linked lists containing 25 million nodes. At some point the clients are stopped - and a map reduce job is run to ensure no linked list has a hole. A hole indicates - data was lost.</description> + linked lists containing 25 million nodes. At some point the clients are stopped + and a map reduce job is run to ensure no linked list has a hole. A hole indicates + data was lost.</description> <organization> <name>The Apache Software Foundation</name> <url>http://www.apache.org/</url> @@ -48,7 +49,7 @@ </ciManagement> <properties> - <jclouds.version>1.7.3</jclouds.version> + <jclouds.version>1.8.0</jclouds.version> </properties> <build> @@ -262,6 +263,13 @@ <groupId>org.apache.gora</groupId> <artifactId>gora-core</artifactId> </dependency> + <dependency> + <groupId>org.apache.gora</groupId> + <artifactId>gora-core-tests</artifactId> + <version>0.5-SNAPSHOT</version> + <scope>system</scope> + <systemPath>${project.basedir}/libext/gora-core-tests-0.5-SNAPSHOT.jar</systemPath> + </dependency> <dependency> <groupId>org.apache.hadoop</groupId> @@ -277,6 +285,12 @@ <artifactId>commons-cli</artifactId> </dependency> + <!-- dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <scope>runtime</scope> + </dependency--> + <!-- jclouds dependencies --> <dependency> <groupId>org.apache.jclouds.driver</groupId> http://git-wip-us.apache.org/repos/asf/gora/blob/3c5bd0ee/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/RackspaceOrchestration.java ---------------------------------------------------------------------- diff --git a/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/RackspaceOrchestration.java b/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/RackspaceOrchestration.java new file mode 100644 index 0000000..b27baaa --- /dev/null +++ b/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/RackspaceOrchestration.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gora.goraci.rackspace; + +import java.io.File; +import java.io.IOException; +import java.util.NoSuchElementException; +import java.util.Properties; + +import org.apache.gora.mock.store.MockDataStore; +import org.apache.gora.store.DataStoreFactory; +import org.jclouds.ContextBuilder; +import org.jclouds.openstack.nova.v2_0.NovaApi; +import org.jclouds.openstack.nova.v2_0.domain.Flavor; +import org.jclouds.openstack.nova.v2_0.domain.Image; +import org.jclouds.openstack.nova.v2_0.domain.KeyPair; +import org.jclouds.openstack.nova.v2_0.domain.ServerCreated; +import org.jclouds.openstack.nova.v2_0.extensions.KeyPairApi; +import org.jclouds.openstack.nova.v2_0.features.FlavorApi; +import org.jclouds.openstack.nova.v2_0.features.ImageApi; +import org.jclouds.openstack.nova.v2_0.features.ServerApi; +import org.jclouds.openstack.nova.v2_0.options.CreateServerOptions; +import org.jclouds.openstack.nova.v2_0.predicates.ServerPredicates; + +import com.google.common.base.Charsets; +import com.google.common.io.Files; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is the main class for initiating Rackspace cloud + * topography for use within the GoraCI job. + * @param <K> + * + */ +public class RackspaceOrchestration<K> { + + private static final Logger LOG = LoggerFactory.getLogger(RackspaceOrchestration.class); + + private static String RS_CONTINENT = "rackspace.continent"; + + private static String RS_USERNAME = "rackspace.username"; + + private static String RS_APIKEY = "rackspace.apikey"; + + private static String RS_REGION = "rackspace.region"; + + private static String RS_IMAGEID = "rackspace.imageid"; + + private static String RS_FLAVORID = "rackspace.flavorid"; + + private static String RS_SERVERNAME = "rackspace.servername"; + + private static String RS_NUM_SERVERS = "rackspace.num.servers"; + + /* we can optionally upload our own keys for security, otherwise + * we can have Rackspace autocreate one for us*/ + private static String RS_PUBKEY = "rackspace.uploadpubkey"; + + /** + * @param args + * @throws IllegalAccessException + * @throws InstantiationException + * @throws NoSuchElementException + */ + public static void main(String[] args) throws NoSuchElementException, InstantiationException, IllegalAccessException { + Properties properties = DataStoreFactory.createProps(); + String rsContinent = DataStoreFactory.findProperty(properties, MockDataStore.class.newInstance(), + RS_CONTINENT, "rackspace-cloudservers-us"); + String rsUser = DataStoreFactory.findProperty(properties, MockDataStore.class.newInstance(), RS_USERNAME, "asf-gora"); + String rsApiKey = DataStoreFactory.findProperty(properties, MockDataStore.class.newInstance(), RS_APIKEY, null); + String rs_region = DataStoreFactory.findProperty(properties, MockDataStore.class.newInstance(), RS_REGION, "DFW"); + String rs_flavourId = DataStoreFactory.findProperty(properties, MockDataStore.class.newInstance(), RS_FLAVORID, null); + String rs_imageId = DataStoreFactory.findProperty(properties, MockDataStore.class.newInstance(), RS_IMAGEID, null); + int num_servers = Integer.parseInt(DataStoreFactory.findProperty(properties, MockDataStore.class.newInstance(), RS_NUM_SERVERS, "10")); + String serverName = DataStoreFactory.findProperty(properties, MockDataStore.class.newInstance(), RS_SERVERNAME, "goraci_test_server"); + + NovaApi novaApi = ContextBuilder.newBuilder(rsContinent).credentials(rsUser, rsApiKey).buildApi(NovaApi.class); + LOG.info("Defining Rackspace cloudserver continent as: {}, and region: {}.", rsContinent, rs_region); + + //Choose operating system + ImageApi imageApi = novaApi.getImageApiForZone(rs_region); + Image image = imageApi.get(rs_imageId); + //Choose hardware configuration + FlavorApi flavorApi = novaApi.getFlavorApiForZone(rs_region); + Flavor flavor = flavorApi.get(rs_flavourId); + LOG.info("Defining Rackspace cloudserver flavors as: {}, with image id's {}", rs_flavourId, rs_imageId); + + //Manage keypairs + KeyPairApi keyPairApi = novaApi.getKeyPairExtensionForZone(rs_region).get(); + + File keyPairFile = null; + String publicKey = null; + //Use your own .pub key which should be on CP + if (DataStoreFactory.findBooleanProperty(properties, MockDataStore.class.newInstance(), RS_PUBKEY, "true")) { + keyPairFile = new File("~/.ssh/id_rsa.pub"); + LOG.info("Uploading local public key from ~/.ssh/id_rsa.pub to Rackspace..."); + } else { + //obtain existing key from Rackspace + if (keyPairApi.get("goraci-keypair") != null) { + publicKey = keyPairApi.get("goraci-keypair").getPublicKey(); + LOG.info("Obtained existing public key 'goraci-keypair' from Rackspace..."); + } else { + //have Rackspace generate us a key + LOG.info("Generating local keypair 'goraci-keypair' and uploading to Rackspace..."); + KeyPair keyPair = keyPairApi.create("goraci-keypair"); + keyPairFile = new File("/target/goraci-keypair.pem"); + try { + Files.write(keyPair.getPrivateKey(), keyPairFile, Charsets.UTF_8); + } catch (IOException e) { + e.printStackTrace(); + } + try { + publicKey = Files.toString(keyPairFile, Charsets.UTF_8); + } catch (IOException e) { + e.printStackTrace(); + } + keyPairApi.createWithPublicKey("goraci-keypair", publicKey); + } + } + + ServerApi serverApi = novaApi.getServerApiForZone(rs_region); + CreateServerOptions options = CreateServerOptions.Builder.keyPairName("goraci-keypair"); + ServerCreated serverCreated = null; + for (int i = 0; i < num_servers; i++) { + if (serverName != null) { + serverCreated = serverApi.create(serverName + num_servers, rs_imageId, rs_flavourId, options); + + } else { + serverCreated = serverApi.create("goraci_test_server" + num_servers, image.getId(), flavor.getId(), options); + } + ServerPredicates.awaitActive(serverApi).apply(serverCreated.getId()); + LOG.info("Provisioned node " + i + 1 + " of " + num_servers + " with name: " + serverName + i); + } + } + +} http://git-wip-us.apache.org/repos/asf/gora/blob/3c5bd0ee/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/package-info.java ---------------------------------------------------------------------- diff --git a/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/package-info.java b/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/package-info.java new file mode 100644 index 0000000..5c526ee --- /dev/null +++ b/gora-goraci/src/main/java/org/apache/gora/goraci/rackspace/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Main package for all Rackspace-enabling functionality for the + * GoraCI test. + * + */ +package org.apache.gora.goraci.rackspace; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/gora/blob/3c5bd0ee/gora-goraci/src/main/resources/gora.properties ---------------------------------------------------------------------- diff --git a/gora-goraci/src/main/resources/gora.properties b/gora-goraci/src/main/resources/gora.properties index 7b4733b..bec6623 100644 --- a/gora-goraci/src/main/resources/gora.properties +++ b/gora-goraci/src/main/resources/gora.properties @@ -13,6 +13,52 @@ # See the License for the specific language governing permissions and # limitations under the License. + +# Available continents are cloudservers-us and cloudservers-uk +gora.datastore.rackspace.continent=rackspace-cloudservers-us +gora.datastore.rackspace.username=asf-gora +gora.datastore.rackspace.apikey= +#Currently you can pick from the following regions +# * US Northern Virginia - IAD=Suppliers.ofInstance(https://iad.servers.api.rackspacecloud.com/v2/910883) +# * US Chicago - ORD=Suppliers.ofInstance(https://ord.servers.api.rackspacecloud.com/v2/910883) +# * US Dallas - DFW=Suppliers.ofInstance(https://dfw.servers.api.rackspacecloud.com/v2/910883) +# * EU London - lon TBC +# * Asia-Pacific Hong Kong - HKG=Suppliers.ofInstance(https://hkg.servers.api.rackspacecloud.com/v2/910883) +# * Asia-Pacific Sydney - SYD=Suppliers.ofInstance(https://syd.servers.api.rackspacecloud.com/v2/910883) +gora.datastore.rackspace.region=DFW +# A list of images can be found at the link provided below +# http://docs.rackspace.com/servers/api/v2/cs-devguide/content/List_Images-d1e4435.html +# In this instance we pick Ubuntu 13.10 (Saucy Salamander) (PVHVM) +gora.datastore.rackspace.imageid=6110edfe-8589-4bb1-aa27-385f12242627 +# Flavor Information can be found here +#http://docs.rackspace.com/servers/api/v2/cs-devguide/content/List_Flavors-d1e4188.html +# Can be one of the following +# * 2 - 512MB Standard Instance +# * 3 - 1GB Standard Instance +# * 4 - 2GB Standard Instance +# * 5 - 4GB Standard Instance +# * 6 - 8GB Standard Instance +# * 7 - 15GB Standard Instance +# * 8 - 30GB Standard Instance +# * onmetal-compute1 - OnMetal Compute v1 +# * onmetal-io1 - OnMetal I/O v1 +# * onmetal-memory1 - OnMetal Memory v1 +# * performance1-1 - 1 GB Performance +# * performance1-2 - 2 GB Performance +# * performance1-4 - 4 GB Performance +# * performance1-8 - 8 GB Performance +# * performance2-15 - 15 GB Performance +# * performance2-30 - 30 GB Performance +# * performance2-60 - 60 GB Performance +# * performance2-90 - 90 GB Performance +# * performance2-120 - 120 GB Performance +gora.datastore.rackspace.flavorid=6 +gora.datastore.rackspace.servername=goraci_test_server +gora.datastore.rackspace.num.servers=20 +# we can optionally upload our own keys for security, otherwise we can have Rackspace +# create one for us +gora.datastore.rackspace.uploadpubkey=false + ############################### # Default Accumulo properties # ############################### @@ -22,15 +68,6 @@ gora.datastore.accumulo.zookeepers=localhost gora.datastore.accumulo.user=root gora.datastore.accumulo.password=secret -############################### -# Default SqlStore properties # -############################### - -#gora.sqlstore.jdbc.driver=org.hsqldb.jdbcDriver -#gora.sqlstore.jdbc.url=jdbc:hsqldb:hsql://localhost/nutchtest -# gora.sqlstore.jdbc.user= -# gora.sqlstore.jdbc.password= - ################################ # Default AvroStore properties # ################################ http://git-wip-us.apache.org/repos/asf/gora/blob/3c5bd0ee/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 4e1f160..825261f 100644 --- a/pom.xml +++ b/pom.xml @@ -635,7 +635,7 @@ <restlet.version>2.2.1</restlet.version> <!-- Misc Dependencies --> - <guava.version>13.0</guava.version> + <guava.version>17.0</guava.version> <commons-lang.version>2.6</commons-lang.version> <jdom.version>1.1.2</jdom.version> <hsqldb.version>2.2.8</hsqldb.version>
