http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseNodeFailureIT.groovy ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseNodeFailureIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseNodeFailureIT.groovy deleted file mode 100644 index cd87fab..0000000 --- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseNodeFailureIT.groovy +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.hbase.funtest - -import org.apache.hadoop.conf.Configuration -import org.apache.slider.api.ClusterDescription -import org.apache.slider.api.RoleKeys -import org.apache.slider.api.StatusKeys -import org.apache.slider.client.SliderClient -import org.apache.slider.common.params.ActionKillContainerArgs -import org.apache.slider.providers.hbase.HBaseKeys -import org.apache.slider.providers.hbase.HBaseTestUtils - -class HBaseNodeFailureIT extends FunctionalHBaseClusterIT { - - - public static final int RESTART_SLEEP_TIME = 5000 - - @Override - String getClusterName() { - return "test_hbase_node_failure" - } - - @Override - String getDescription() { - "Fail containers and verify that the cluster recovers" - } - - @Override - void clusterOperations( - String clustername, - SliderClient sliderClient, - Configuration clientConf, - int numWorkers, - Map<String, Integer> roleMap, - ClusterDescription cd) { - - - killInstanceOfRole(sliderClient, HBaseKeys.ROLE_WORKER) - // let it take - sleep(RESTART_SLEEP_TIME) - - //wait for the role counts to be reached - cd = waitForRoleCount(sliderClient, roleMap, HBASE_LAUNCH_WAIT_TIME) - // then expect a restart - HBaseTestUtils.waitForHBaseRegionServerCount( - sliderClient, - clusterName, - numWorkers, - HBASE_LAUNCH_WAIT_TIME) - assert cd.roles[HBaseKeys.ROLE_WORKER][RoleKeys.ROLE_FAILED_INSTANCES] == "1" - killInstanceOfRole(sliderClient, HBaseKeys.ROLE_WORKER) - // let it take - sleep(RESTART_SLEEP_TIME) - // then expect a restart - - //wait for the role counts to be reached - cd = waitForRoleCount(sliderClient, roleMap, HBASE_LAUNCH_WAIT_TIME) - - HBaseTestUtils.waitForHBaseRegionServerCount( - sliderClient, - clusterName, - numWorkers, - HBASE_LAUNCH_WAIT_TIME) - assert cd.roles[HBaseKeys.ROLE_WORKER][RoleKeys.ROLE_FAILED_INSTANCES] == "2" - - killInstanceOfRole(sliderClient, HBaseKeys.ROLE_MASTER) - // let it take - sleep(RESTART_SLEEP_TIME) - - // wait for the role counts to be reached - cd = waitForRoleCount(sliderClient, roleMap, HBASE_LAUNCH_WAIT_TIME) - HBaseTestUtils.waitForHBaseRegionServerCount( - sliderClient, - clusterName, - numWorkers, - HBASE_LAUNCH_WAIT_TIME) - assert cd.roles[HBaseKeys.ROLE_MASTER][RoleKeys.ROLE_FAILED_INSTANCES] == "1" - - // now trigger AM failure - ClusterDescription status = killAmAndWaitForRestart(sliderClient, clusterName) - - def restarted = status.getInfo( - StatusKeys.INFO_CONTAINERS_AM_RESTART) - assert restarted != null - assert Integer.parseInt(restarted) == 1 + numWorkers - - } - - /** - * Kill a random in instance of a role in the cluster - * @param sliderClient client - * @param role - * @return ID of container killed - */ - public String killInstanceOfRole( - SliderClient sliderClient, String role) { - ClusterDescription cd = sliderClient.getClusterDescription() - def instances = cd.instances[role] - if (instances == null || instances.size() == 0) { - log.info("No instances of role $role to kill") - return null; - } - String id = instances[new Random().nextInt(instances.size())] - ActionKillContainerArgs args = new ActionKillContainerArgs() - args.id = id - sliderClient.actionKillContainer(clusterName, args) - return id; - } - - - public int getWorkerPortAssignment() { - return 0 - } - - public int getMasterPortAssignment() { - return 0 - } -}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/ImagesIT.groovy ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/ImagesIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/ImagesIT.groovy deleted file mode 100644 index d1b5c55..0000000 --- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/ImagesIT.groovy +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.hbase.funtest - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.FileSystem as HadoopFS -import org.apache.hadoop.fs.Path -import org.apache.slider.funtest.framework.CommandTestBase -import org.apache.slider.funtest.framework.FuntestProperties -import org.junit.Before -import org.junit.Test - -class ImagesIT extends CommandTestBase implements FuntestProperties { - - - @Before - public void verifyPreconditions() { - assumeBoolOption(SLIDER_CONFIG, KEY_TEST_HBASE_ENABLED, true) - } - - @Test - public void testImageExists() throws Throwable { - - Configuration conf = loadSliderConf() - String testImage = conf.get(KEY_TEST_HBASE_TAR) - assert testImage - Path path = new Path(testImage) - HadoopFS fs = HadoopFS.get( - path.toUri(), - conf) - assert fs.exists(path) - } - - @Test - public void testAppConfExists() throws Throwable { - Configuration conf = loadSliderConf() - String dir = conf.get(KEY_TEST_HBASE_APPCONF) - - assert conf.get(KEY_TEST_HBASE_APPCONF) - Path path = new Path(dir) - HadoopFS fs = HadoopFS.get( - path.toUri(), - conf) - assert fs.exists(path) - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/hbase-funtests/src/test/java/org/apache/slider/providers/hbase/funtest/StubCompile.java ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/hbase-funtests/src/test/java/org/apache/slider/providers/hbase/funtest/StubCompile.java b/slider-providers/hbase/hbase-funtests/src/test/java/org/apache/slider/providers/hbase/funtest/StubCompile.java deleted file mode 100644 index d426635..0000000 --- a/slider-providers/hbase/hbase-funtests/src/test/java/org/apache/slider/providers/hbase/funtest/StubCompile.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.hbase.funtest; - -class StubCompile { -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/hbase-funtests/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/hbase-funtests/src/test/resources/log4j.properties b/slider-providers/hbase/hbase-funtests/src/test/resources/log4j.properties deleted file mode 100644 index a552a55..0000000 --- a/slider-providers/hbase/hbase-funtests/src/test/resources/log4j.properties +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# log4j configuration used during build and unit tests - -log4j.rootLogger=INFO,stdout -log4j.threshhold=ALL -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n - -log4j.appender.subprocess=org.apache.log4j.ConsoleAppender -log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout -log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n -#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess - -log4j.logger.org.apache.slider=DEBUG -log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG - - - -#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG -#log4j.logger.org.apache.hadoop.yarn.service=DEBUG -#log4j.logger.org.apache.hadoop.yarn.client=DEBUG - -#crank back on some noise -log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR -log4j.logger.org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner=WARN -log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN -log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN -log4j.logger.org.apache.hadoop.hdfs=WARN - - -log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN -log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN -log4j.logger.org.apache.zookeeper=WARN -log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL - -log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.security=WARN -log4j.logger.org.apache.hadoop.metrics2=ERROR -log4j.logger.org.apache.hadoop.util.HostsFileReader=WARN -log4j.logger.org.apache.hadoop.yarn.event.AsyncDispatcher=WARN -log4j.logger.org.apache.hadoop.security.token.delegation=WARN -log4j.logger.org.apache.hadoop.yarn.util.AbstractLivelinessMonitor=WARN -log4j.logger.org.apache.hadoop.yarn.server.nodemanager.security=WARN -log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMNMInfo=WARN http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/pom.xml ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/pom.xml b/slider-providers/hbase/slider-hbase-provider/pom.xml deleted file mode 100644 index 7453a70..0000000 --- a/slider-providers/hbase/slider-hbase-provider/pom.xml +++ /dev/null @@ -1,256 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - <modelVersion>4.0.0</modelVersion> - <artifactId>slider-hbase-provider</artifactId> - <name>Slider HBase Provider</name> - <packaging>jar</packaging> - <description> - Direct provider for slider. This is the original provider from Hoya, moved to one side - as it is no longer being actively developed. It contains tests against - mini clusters as well as full functional tests - - </description> - <parent> - <groupId>org.apache.slider</groupId> - <artifactId>slider</artifactId> - <version>0.61-incubating</version> - <relativePath>../../../</relativePath> - </parent> - - <build> - - <!-- resources are filtered for dynamic updates. This gets build info in--> - <resources> - <resource> - <directory>src/main/resources</directory> - <filtering>true</filtering> - </resource> - </resources> - - <plugins> - - <plugin> - <artifactId>maven-compiler-plugin</artifactId> - <dependencies> - <dependency> - <groupId>org.codehaus.groovy</groupId> - <artifactId>groovy-eclipse-compiler</artifactId> - <version>${groovy-eclipse-compiler.version}</version> - </dependency> - <dependency> - <groupId>org.codehaus.groovy</groupId> - <artifactId>groovy-eclipse-batch</artifactId> - <version>${groovy-eclipse-batch.version}</version> - </dependency> - </dependencies> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <version>${maven-jar-plugin.version}</version> - <executions> - <execution> - <goals> - <goal>test-jar</goal> - </goals> - </execution> - </executions> - </plugin> - - <!-- test --> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <version>${maven-surefire-plugin.version}</version> - <configuration> - <!--mvn process fork options--> - <reuseForks>${test.reuseForks}</reuseForks> - <forkMode>${test.forkMode}</forkMode> - <forkCount>1</forkCount> - <forkedProcessTimeoutInSeconds>${test.forkedProcessTimeoutInSeconds}</forkedProcessTimeoutInSeconds> - <threadCount>1</threadCount> - <argLine>${test.argLine}</argLine> - <failIfNoTests>${test.failIfNoTests}</failIfNoTests> - - <trimStackTrace>false</trimStackTrace> - <redirectTestOutputToFile>${build.redirect.test.output.to.file}</redirectTestOutputToFile> - <systemPropertyVariables> - <java.net.preferIPv4Stack>true</java.net.preferIPv4Stack> - <java.awt.headless>true</java.awt.headless> - <java.security.krb5.realm>${slider.test.java.security.krb5.realm}</java.security.krb5.realm> - <java.security.krb5.kdc>${slider.test.java.security.krb5.kdc}</java.security.krb5.kdc> - </systemPropertyVariables> - <includes> - <include>**/Test*.java</include> - </includes> - <excludes> - <exclude>**/Test*$*.java</exclude> - </excludes> - </configuration> - </plugin> - - - - </plugins> - </build> - - <reporting> - <plugins> - - - - </plugins> - </reporting> - - <dependencies> - - <dependency> - <groupId>org.apache.slider</groupId> - <artifactId>slider-core</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.slider</groupId> - <artifactId>slider-core</artifactId> - <type>test-jar</type> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.codehaus.groovy</groupId> - <artifactId>groovy-all</artifactId> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-client</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-minicluster</artifactId> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-server</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-protocol</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-common</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-common</artifactId> - <classifier>tests</classifier> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-it</artifactId> - <classifier>tests</classifier> - <exclusions> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - </exclusion> - </exclusions> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-hadoop-compat</artifactId> - <classifier>tests</classifier> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-hadoop2-compat</artifactId> - <classifier>tests</classifier> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-server</artifactId> - <classifier>tests</classifier> - <scope>test</scope> - </dependency> - - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </dependency> - - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - - - </dependencies> - - <profiles> - <profile> - <id>rat</id> - <build> - <plugins> - <plugin> - <groupId>org.apache.rat</groupId> - <artifactId>apache-rat-plugin</artifactId> - <version>${apache-rat-plugin.version}</version> - <executions> - <execution> - <id>check-licenses</id> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> - <configuration> - <excludes> - <exclude>**/*.json</exclude> - <exclude>src/main/resources/org/apache/slider/providers/hbase/conf/regionservers</exclude> - </excludes> - </configuration> - </plugin> - </plugins> - </build> - </profile> - </profiles> -</project> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java deleted file mode 100644 index e0acd06..0000000 --- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java +++ /dev/null @@ -1,337 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.hbase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.slider.api.InternalKeys; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; -import org.apache.slider.api.OptionKeys; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.conf.MapOperations; -import org.apache.slider.core.launch.AbstractLauncher; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.providers.AbstractClientProvider; -import org.apache.slider.providers.ProviderRole; -import org.apache.slider.providers.ProviderUtils; -import org.apache.slider.common.tools.ConfigHelper; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.zk.ZookeeperUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * This class implements the client-side aspects - * of an HBase Cluster - */ -public class HBaseClientProvider extends AbstractClientProvider implements - HBaseKeys, SliderKeys, - HBaseConfigFileOptions { - - - protected static final Logger log = - LoggerFactory.getLogger(HBaseClientProvider.class); - protected static final String NAME = "hbase"; - private static final String INSTANCE_RESOURCE_BASE = "/org/apache/slider/providers/hbase/instance/"; - - - protected HBaseClientProvider(Configuration conf) { - super(conf); - } - - @Override - public String getName() { - return NAME; - } - - @Override - public List<ProviderRole> getRoles() { - return HBaseRoles.getRoles(); - } - - - @Override - public void prepareInstanceConfiguration(AggregateConf aggregateConf) throws - SliderException, - IOException { - String resourceTemplate = INSTANCE_RESOURCE_BASE + "resources.json"; - String appConfTemplate = INSTANCE_RESOURCE_BASE + "appconf.json"; - mergeTemplates(aggregateConf, null, resourceTemplate, appConfTemplate); - } - - /** - * Build the hdfs-site.xml file - * This the configuration used by HBase directly - * @param instanceDescription this is the cluster specification used to define this - * @return a map of the dynamic bindings for this Slider instance - */ - public Map<String, String> buildSiteConfFromInstance( - AggregateConf instanceDescription) - throws BadConfigException { - - - ConfTreeOperations appconf = - instanceDescription.getAppConfOperations(); - - MapOperations globalAppOptions = appconf.getGlobalOptions(); - MapOperations globalInstanceOptions = instanceDescription.getInternalOperations().getGlobalOptions(); - MapOperations master = appconf.getMandatoryComponent(HBaseKeys.ROLE_MASTER); - - MapOperations worker = appconf.getMandatoryComponent(HBaseKeys.ROLE_WORKER); - - Map<String, String> sitexml = new HashMap<String, String>(); - - //map all cluster-wide site. options - providerUtils.propagateSiteOptions(globalAppOptions, sitexml); -/* - //this is where we'd do app-indepdenent keytabs - - String keytab = - clusterSpec.getOption(OptionKeys.OPTION_KEYTAB_LOCATION, ""); - -*/ - - - sitexml.put(KEY_HBASE_ROOTDIR, - globalInstanceOptions.getMandatoryOption( - InternalKeys.INTERNAL_DATA_DIR_PATH) - ); - providerUtils.propagateOption(globalAppOptions, OptionKeys.ZOOKEEPER_PATH, - sitexml, KEY_ZNODE_PARENT); - String quorum = - globalAppOptions.getMandatoryOption(OptionKeys.ZOOKEEPER_QUORUM); - sitexml.put(KEY_ZOOKEEPER_QUORUM, ZookeeperUtils.convertToHostsOnlyList(quorum)); - //and assume first port works everywhere - sitexml.put(KEY_ZOOKEEPER_PORT, - Integer.toString(ZookeeperUtils.getFirstPort(quorum, HBASE_ZK_PORT))); - - return sitexml; - } - - @Override //Client - public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem, - String clustername, - Configuration configuration, - AggregateConf instanceDefinition, - Path clusterDirPath, - Path generatedConfDirPath, - boolean secure) throws - SliderException, - IOException { - super.preflightValidateClusterConfiguration(sliderFileSystem, clustername, - configuration, - instanceDefinition, - clusterDirPath, - generatedConfDirPath, secure); - - Path templatePath = new Path(generatedConfDirPath, HBaseKeys.SITE_XML); - //load the HBase site file or fail - Configuration siteConf = ConfigHelper.loadConfiguration(sliderFileSystem.getFileSystem(), - templatePath); - - //core customizations - validateHBaseSiteXML(siteConf, secure, templatePath.toString()); - - } - - /** - * Validate the hbase-site.xml values - * @param siteConf site config - * @param secure secure flag - * @param origin origin for exceptions - * @throws BadConfigException if a config is missing/invalid - */ - public void validateHBaseSiteXML(Configuration siteConf, - boolean secure, - String origin) throws BadConfigException { - try { - SliderUtils.verifyOptionSet(siteConf, KEY_HBASE_CLUSTER_DISTRIBUTED, - false); - SliderUtils.verifyOptionSet(siteConf, KEY_HBASE_ROOTDIR, false); - SliderUtils.verifyOptionSet(siteConf, KEY_ZNODE_PARENT, false); - SliderUtils.verifyOptionSet(siteConf, KEY_ZOOKEEPER_QUORUM, false); - SliderUtils.verifyOptionSet(siteConf, KEY_ZOOKEEPER_PORT, false); - int zkPort = - siteConf.getInt(HBaseConfigFileOptions.KEY_ZOOKEEPER_PORT, 0); - if (zkPort == 0) { - throw new BadConfigException( - "ZK port property not provided at %s in configuration file %s", - HBaseConfigFileOptions.KEY_ZOOKEEPER_PORT, - siteConf); - } - - if (secure) { - //better have the secure cluster definition up and running - SliderUtils.verifyOptionSet(siteConf, KEY_MASTER_KERBEROS_PRINCIPAL, - false); - SliderUtils.verifyOptionSet(siteConf, KEY_MASTER_KERBEROS_KEYTAB, - false); - SliderUtils.verifyOptionSet(siteConf, - KEY_REGIONSERVER_KERBEROS_PRINCIPAL, - false); - SliderUtils.verifyOptionSet(siteConf, - KEY_REGIONSERVER_KERBEROS_KEYTAB, false); - } - } catch (BadConfigException e) { - //bad configuration, dump it - - log.error("Bad site configuration {} : {}", origin, e, e); - log.info(ConfigHelper.dumpConfigToString(siteConf)); - throw e; - } - } - - private static Set<String> knownRoleNames = new HashSet<String>(); - static { - List<ProviderRole> roles = HBaseRoles.getRoles(); - knownRoleNames.add(SliderKeys.COMPONENT_AM); - for (ProviderRole role : roles) { - knownRoleNames.add(role.name); - } - } - - /** - * Validate the instance definition. - * @param instanceDefinition instance definition - */ - @Override - public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws - SliderException { - super.validateInstanceDefinition(instanceDefinition, fs); - ConfTreeOperations resources = - instanceDefinition.getResourceOperations(); - Set<String> unknownRoles = resources.getComponentNames(); - unknownRoles.removeAll(knownRoleNames); - if (!unknownRoles.isEmpty()) { - throw new BadCommandArgumentsException("Unknown component: %s", - unknownRoles.iterator().next()); - } - providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_WORKER, - 0, -1); - providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_MASTER, - 0, -1); - - providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_REST_GATEWAY, - 0, -1); - providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_THRIFT_GATEWAY, - 0, -1); - providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_THRIFT2_GATEWAY, - 0, -1); - } - - @Override - public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem, - Configuration serviceConf, - AbstractLauncher launcher, - AggregateConf instanceDescription, - Path snapshotConfDirPath, - Path generatedConfDirPath, - Configuration clientConfExtras, - String libdir, - Path tempPath, boolean miniClusterTestRun) throws - IOException, - SliderException { - - // add any and all dependency files - Map<String, LocalResource> providerResources = - new HashMap<String, LocalResource>(); - - ProviderUtils.addProviderJar(providerResources, - this, - "slider-hbase-provider.jar", - fileSystem, - tempPath, - libdir, miniClusterTestRun); - - Class<?>[] classes = { - }; - String[] jars = - { - }; - ProviderUtils.addDependencyJars(providerResources, fileSystem, tempPath, - libdir, jars, - classes); - - - launcher.addLocalResources(providerResources); - - //load in the template site config - log.debug("Loading template configuration from {}", snapshotConfDirPath); - Configuration siteConf = ConfigHelper.loadTemplateConfiguration( - serviceConf, - snapshotConfDirPath, - HBaseKeys.SITE_XML, - HBaseKeys.HBASE_TEMPLATE_RESOURCE); - - if (log.isDebugEnabled()) { - log.debug("Configuration came from {}", - siteConf.get(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN)); - ConfigHelper.dumpConf(siteConf); - } - //construct the cluster configuration values - - ConfTreeOperations appconf = - instanceDescription.getAppConfOperations(); - - - Map<String, String> clusterConfMap = buildSiteConfFromInstance( - instanceDescription); - - //merge them - ConfigHelper.addConfigMap(siteConf, - clusterConfMap.entrySet(), - "HBase Provider"); - - //now, if there is an extra client conf, merge it in too - if (clientConfExtras != null) { - ConfigHelper.mergeConfigurations(siteConf, clientConfExtras, - "Slider Client", true); - } - - if (log.isDebugEnabled()) { - log.debug("Merged Configuration"); - ConfigHelper.dumpConf(siteConf); - } - - Path sitePath = ConfigHelper.saveConfig(serviceConf, - siteConf, - generatedConfDirPath, - HBaseKeys.SITE_XML); - - log.debug("Saving the config to {}", sitePath); - launcher.submitDirectory(generatedConfDirPath, - SliderKeys.PROPAGATED_CONF_DIR_NAME); - - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseConfigFileOptions.java ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseConfigFileOptions.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseConfigFileOptions.java deleted file mode 100644 index ac1a6ac..0000000 --- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseConfigFileOptions.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.hbase; - -/** - * Mappings of config params to env variables for - * custom -site.xml files to pick up - * - * A lot of these come from HConstants -the reason they have been copied - * and pasted in here is to remove dependencies on HBase from - * the Slider Client and AM. - */ -public interface HBaseConfigFileOptions { - - String KEY_HBASE_CLUSTER_DISTRIBUTED = "hbase.cluster.distributed"; - String KEY_HBASE_ROOTDIR = "hbase.rootdir"; - - String KEY_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; - //HConstants.ZOOKEEPER_QUORUM; - String KEY_ZOOKEEPER_PORT = "hbase.zookeeper.property.clientPort"; - //HConstants.ZOOKEEPER_CLIENT_PORT; - String KEY_ZNODE_PARENT = "zookeeper.znode.parent"; - - - int DEFAULT_MASTER_PORT = 60000; - int DEFAULT_MASTER_INFO_PORT = 60010; - - String KEY_HBASE_MASTER_PORT = "hbase.master.port"; - String KEY_HBASE_MASTER_INFO_PORT = "hbase.master.info.port"; - - int HBASE_ZK_PORT = 2181; // HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT; - - - String KEY_REGIONSERVER_PORT = "hbase.regionserver.port"; - String KEY_REGIONSERVER_INFO_PORT = "hbase.regionserver.info.port"; - - /** - * needed to relax constraints in negotiations, including tests - */ - String IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH = - "ipc.client.fallback-to-simple-auth-allowed"; - - /* - <property> - <name>hbase.regionserver.kerberos.principal</name> - <value>hbase/[email protected]</value> - </property> - <property> - <name>hbase.regionserver.keytab.file</name> - <value>/etc/hbase/conf/keytab.krb5</value> - </property> - <property> - <name>hbase.master.kerberos.principal</name> - <value>hbase/[email protected]</value> - </property> - <property> - <name>hbase.master.keytab.file</name> - <value>/etc/hbase/conf/keytab.krb5</value> - </property> - */ - - - String KEY_REGIONSERVER_KERBEROS_PRINCIPAL = "hbase.regionserver.kerberos.principal"; - String KEY_REGIONSERVER_KERBEROS_KEYTAB = "hbase.regionserver.keytab.file"; - - String KEY_MASTER_KERBEROS_PRINCIPAL = "hbase.master.kerberos.principal"; - String KEY_MASTER_KERBEROS_KEYTAB = "hbase.master.keytab.file"; - - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java deleted file mode 100644 index 064414e..0000000 --- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.hbase; - -public interface HBaseKeys { - - /** {@value */ - String MASTER = "master"; - String ROLE_WORKER = "worker"; - - String ROLE_MASTER = MASTER; - - String ROLE_REST_GATEWAY = "REST"; - - String ROLE_THRIFT_GATEWAY = "THRIFT"; - - String ROLE_THRIFT2_GATEWAY = "THRIFT2"; - - /** {@value */ - String REGION_SERVER = "regionserver"; - - /** {@value */ - String REST_GATEWAY = "rest"; - - /** {@value */ - String THRIFT_GATEWAY = "thrift"; - /** {@value */ - String THRIFT2_GATEWAY = "thrift2"; - /** - /** - * What is the command for hbase to print a version: {@value} - */ - String COMMAND_VERSION = "version"; - - String ACTION_START = "start"; - String ACTION_STOP = "stop"; - - /** - * Config directory : {@value} - */ - String ARG_CONFIG = "--config"; - /** - * name of the hbase script relative to the hbase root dir: {@value} - */ - String HBASE_SCRIPT = "hbase"; - - /** - * name of the site conf to generate : {@value} - */ - String SITE_XML = "hbase-site.xml"; - /** - * Template stored in the slider classpath -to use if there is - * no site-specific template - * {@value} - */ - String HBASE_CONF_RESOURCE = "org/apache/slider/providers/hbase/conf/"; - String HBASE_TEMPLATE_RESOURCE = HBASE_CONF_RESOURCE + SITE_XML; - - - String DEFAULT_HBASE_WORKER_HEAP = "512M"; - String DEFAULT_HBASE_MASTER_HEAP = "512M"; - String DEFAULT_HBASE_WORKER_INFOPORT = "0"; - String DEFAULT_HBASE_MASTER_INFOPORT = "0"; - String PROVIDER_HBASE = "hbase"; - String HBASE_LOG_DIR = "HBASE_LOG_DIR"; - - String HBASE_HEAPSIZE = "HBASE_HEAPSIZE"; - String HBASE_GC_OPTS = "SERVER_GC_OPTS"; - - String PROPAGATED_CONFDIR = "PROPAGATED_CONFDIR"; - - /** - * Service type used in registry - */ - String HBASE_SERVICE_TYPE = "org-apache-hbase"; - String HBASE_SITE_PUBLISHED_CONFIG = "hbase-site"; -} - - http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderFactory.java ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderFactory.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderFactory.java deleted file mode 100644 index aa10cbd..0000000 --- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderFactory.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.hbase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.slider.providers.AbstractClientProvider; -import org.apache.slider.providers.SliderProviderFactory; -import org.apache.slider.providers.ProviderService; - -public class HBaseProviderFactory extends SliderProviderFactory { - - public HBaseProviderFactory() { - } - - public HBaseProviderFactory(Configuration conf) { - super(conf); - } - - @Override - public AbstractClientProvider createClientProvider() { - return new HBaseClientProvider(getConf()); - } - - @Override - public ProviderService createServerProvider() { - return new HBaseProviderService(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java deleted file mode 100644 index 3b6cf8b..0000000 --- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java +++ /dev/null @@ -1,368 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.hbase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; -import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; -import org.apache.slider.api.InternalKeys; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.api.ClusterDescription; -import org.apache.slider.api.RoleKeys; -import org.apache.slider.api.StatusKeys; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.MapOperations; -import org.apache.slider.core.launch.CommandLineBuilder; -import org.apache.slider.core.launch.ContainerLauncher; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.exceptions.SliderInternalStateException; -import org.apache.slider.core.registry.docstore.PublishedConfigSet; -import org.apache.slider.core.registry.docstore.PublishedConfiguration; -import org.apache.slider.core.registry.info.CustomRegistryConstants; -import org.apache.slider.providers.AbstractProviderService; -import org.apache.slider.providers.ProviderCore; -import org.apache.slider.providers.ProviderRole; -import org.apache.slider.providers.ProviderUtils; -import org.apache.slider.common.tools.ConfigHelper; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations; -import org.apache.slider.server.appmaster.web.rest.agent.HeartBeat; -import org.apache.slider.server.appmaster.web.rest.agent.HeartBeatResponse; -import org.apache.slider.server.appmaster.web.rest.agent.Register; -import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse; -import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; - -import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_PUBLISHER; - -/** - * This class implements the server-side aspects - * of an HBase Cluster - */ -public class HBaseProviderService extends AbstractProviderService - implements ProviderCore, HBaseKeys, SliderKeys, AgentRestOperations{ - - protected static final Logger log = - LoggerFactory.getLogger(HBaseProviderService.class); - private static final ProviderUtils providerUtils = new ProviderUtils(log); - private HBaseClientProvider clientProvider; - private Configuration siteConf; - - public HBaseProviderService() { - super("HBaseProviderService"); - setAgentRestOperations(this); - } - - @Override - public List<ProviderRole> getRoles() { - return HBaseRoles.getRoles(); - } - - @Override - protected void serviceInit(Configuration conf) throws Exception { - super.serviceInit(conf); - clientProvider = new HBaseClientProvider(conf); - } - - @Override - public Configuration loadProviderConfigurationInformation(File confDir) - throws BadCommandArgumentsException, IOException { - - return loadProviderConfigurationInformation(confDir, SITE_XML); - } - - /** - * Validate the cluster specification. This can be invoked on both - * server and client - * @param instanceDefinition the instance definition to validate - */ - @Override // Client and Server - public void validateInstanceDefinition(AggregateConf instanceDefinition) - throws SliderException { - clientProvider.validateInstanceDefinition(instanceDefinition, null); - } - - @Override - public void buildContainerLaunchContext(ContainerLauncher launcher, - AggregateConf instanceDefinition, - Container container, - String role, - SliderFileSystem coreFS, - Path generatedConfPath, - MapOperations resourceComponent, - MapOperations appComponent, - Path containerTmpDirPath) throws IOException, SliderException { - - // Set the environment - launcher.putEnv(SliderUtils.buildEnvMap(appComponent)); - - String logDirs = providerUtils.getLogdir(); - String logDir; - int idx = logDirs.indexOf(","); - if (idx > 0) { - // randomly choose a log dir candidate - String[] segments = logDirs.split(","); - Random rand = new Random(); - logDir = segments[rand.nextInt(segments.length)]; - } else logDir = logDirs; - launcher.setEnv(HBASE_LOG_DIR, logDir); - - launcher.setEnv(PROPAGATED_CONFDIR, - ProviderUtils.convertToAppRelativePath( - SliderKeys.PROPAGATED_CONF_DIR_NAME) ); - - //local resources - - //add the configuration resources - launcher.addLocalResources(coreFS.submitDirectory( - generatedConfPath, - SliderKeys.PROPAGATED_CONF_DIR_NAME)); - //Add binaries - //now add the image if it was set - String imageURI = instanceDefinition.getInternalOperations() - .get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); - coreFS.maybeAddImagePath(launcher.getLocalResources(), imageURI); - - CommandLineBuilder cli = new CommandLineBuilder(); - - String heap = appComponent.getOption(RoleKeys.JVM_HEAP, DEFAULT_JVM_HEAP); - if (SliderUtils.isSet(heap)) { - String adjustedHeap = SliderUtils.translateTrailingHeapUnit(heap); - launcher.setEnv(HBASE_HEAPSIZE, adjustedHeap); - } - - String gcOpts = appComponent.getOption(RoleKeys.GC_OPTS, DEFAULT_GC_OPTS); - if (SliderUtils.isSet(gcOpts)) { - launcher.setEnv(HBASE_GC_OPTS, gcOpts); - } - - //this must stay relative if it is an image - cli.add(providerUtils.buildPathToScript( - instanceDefinition, - "bin", - HBASE_SCRIPT)); - //config dir is relative to the generated file - cli.add(ARG_CONFIG); - cli.add("$PROPAGATED_CONFDIR"); - - String roleCommand; - String logfile; - //now look at the role - -/* JDK7 - switch (role) { - case ROLE_WORKER: - //role is region server - roleCommand = REGION_SERVER; - logfile = "/region-server.txt"; - break; - case ROLE_MASTER: - roleCommand = MASTER; - - logfile = "/master.txt"; - break; - case ROLE_REST_GATEWAY: - roleCommand = REST_GATEWAY; - - logfile = "/rest-gateway.txt"; - break; - case ROLE_THRIFT_GATEWAY: - roleCommand = THRIFT_GATEWAY; - - logfile = "/thrift-gateway.txt"; - break; - case ROLE_THRIFT2_GATEWAY: - roleCommand = THRIFT2_GATEWAY; - - logfile = "/thrift2-gateway.txt"; - break; - default: - throw new SliderInternalStateException("Cannot start role %s", role); - } - -*/ - if (ROLE_WORKER.equals(role)) { - //role is region server - roleCommand = REGION_SERVER; - logfile = "/region-server.txt"; - - } else if (ROLE_MASTER.equals(role)) { - roleCommand = MASTER; - logfile = "/master.txt"; - - } else if (ROLE_REST_GATEWAY.equals(role)) { - roleCommand = REST_GATEWAY; - logfile = "/rest-gateway.txt"; - - } else if (ROLE_THRIFT_GATEWAY.equals(role)) { - roleCommand = THRIFT_GATEWAY; - logfile = "/thrift-gateway.txt"; - - } else if (ROLE_THRIFT2_GATEWAY.equals(role)) { - roleCommand = THRIFT2_GATEWAY; - logfile = "/thrift2-gateway.txt"; - } - - else { - throw new SliderInternalStateException("Cannot start role %s", role); - } - - cli.add(roleCommand); - cli.add(ACTION_START); - //log details - cli.addOutAndErrFiles(logfile, null); - launcher.addCommand(cli.build()); - - } - - @Override - protected void serviceStart() throws Exception { - registerHBaseServiceEntry(); - super.serviceStart(); - } - - private void registerHBaseServiceEntry() throws IOException { - - String name = amState.getApplicationName() ; - ServiceRecord serviceRecord = new ServiceRecord(); - // bond lifespan to the application - serviceRecord.set(YarnRegistryAttributes.YARN_ID, - yarnRegistry.getApplicationAttemptId() - .getApplicationId().toString()); - serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE, - PersistencePolicies.APPLICATION); - try { - URL configURL = new URL(amWebAPI, - SLIDER_PATH_PUBLISHER + "/" + HBASE_SERVICE_TYPE); - - serviceRecord.addExternalEndpoint( - RegistryTypeUtils.restEndpoint( - CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API, - configURL.toURI())); - } catch (URISyntaxException e) { - log.warn("failed to create config URL: {}", e, e); - } - log.info("registering {}/{}", name, HBASE_SERVICE_TYPE); - yarnRegistry.putService(HBASE_SERVICE_TYPE, name, serviceRecord, true); - - PublishedConfiguration publishedSite = - new PublishedConfiguration("HBase site", siteConf); - PublishedConfigSet configSet = - amState.getOrCreatePublishedConfigSet(HBASE_SERVICE_TYPE); - - configSet.put(HBASE_SITE_PUBLISHED_CONFIG, publishedSite); - } - - /** - * This is a validation of the application configuration on the AM. - * Here is where things like the existence of keytabs and other - * not-seen-client-side properties can be tested, before - * the actual process is spawned. - * @param instanceDefinition clusterSpecification - * @param confDir configuration directory - * @param secure flag to indicate that secure mode checks must exist - * @throws IOException IO problemsn - * @throws SliderException any failure - */ - @Override - public void validateApplicationConfiguration(AggregateConf instanceDefinition, - File confDir, - boolean secure - ) throws IOException, - SliderException { - String siteXMLFilename = SITE_XML; - File siteXML = new File(confDir, siteXMLFilename); - if (!siteXML.exists()) { - throw new BadCommandArgumentsException( - "Configuration directory %s doesn't contain %s - listing is %s", - confDir, siteXMLFilename, SliderUtils.listDir(confDir)); - } - - //now read it in - siteConf = ConfigHelper.loadConfFromFile(siteXML); - //look in the site spec to see that it is OK - clientProvider.validateHBaseSiteXML(siteConf, secure, siteXMLFilename); - - if (secure) { - //secure mode: take a look at the keytab of master and RS - SliderUtils.verifyKeytabExists(siteConf, - HBaseConfigFileOptions.KEY_MASTER_KERBEROS_KEYTAB); - SliderUtils.verifyKeytabExists(siteConf, - HBaseConfigFileOptions.KEY_REGIONSERVER_KERBEROS_KEYTAB); - - } - } - - - /** - * Build the provider status, can be empty - * @return the provider status - map of entries to add to the info section - */ - public Map<String, String> buildProviderStatus() { - Map<String, String> stats = new HashMap<String, String>(); - - return stats; - } - - - @Override - public Map<String, String> buildMonitorDetails(ClusterDescription clusterDesc) { - Map<String, String> details = super.buildMonitorDetails(clusterDesc); - - details.put("Active HBase Master (RPC): " - + getInfoAvoidingNull(clusterDesc, StatusKeys.INFO_MASTER_ADDRESS),""); - - return details; - } - - @Override - public RegistrationResponse handleRegistration(Register registration) { - // dummy impl - RegistrationResponse response = new RegistrationResponse(); - response.setResponseStatus(RegistrationStatus.OK); - return response; - } - - @Override - public HeartBeatResponse handleHeartBeat(HeartBeat heartBeat) { - // dummy impl - long id = heartBeat.getResponseId(); - HeartBeatResponse response = new HeartBeatResponse(); - response.setResponseId(id + 1L); - return response; - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java deleted file mode 100644 index b2825b8..0000000 --- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.hbase; - -import org.apache.slider.common.SliderKeys; -import org.apache.slider.providers.PlacementPolicy; -import org.apache.slider.providers.ProviderRole; - -import java.util.ArrayList; -import java.util.List; - -public class HBaseRoles { - - /** - * List of roles - */ - protected static final List<ProviderRole> ROLES = - new ArrayList<ProviderRole>(); - - public static final int KEY_WORKER = SliderKeys.ROLE_AM_PRIORITY_INDEX + 1; - - public static final int KEY_MASTER = SliderKeys.ROLE_AM_PRIORITY_INDEX + 2; - - public static final int KEY_REST_GATEWAY = SliderKeys.ROLE_AM_PRIORITY_INDEX + 3; - - public static final int KEY_THRIFT_GATEWAY = SliderKeys.ROLE_AM_PRIORITY_INDEX + 4; - - public static final int KEY_THRIFT2_GATEWAY = SliderKeys.ROLE_AM_PRIORITY_INDEX + 5; - /** - * Initialize role list - */ - static { - ROLES.add(new ProviderRole(HBaseKeys.ROLE_WORKER, KEY_WORKER)); - ROLES.add(new ProviderRole(HBaseKeys.ROLE_MASTER, KEY_MASTER)); - ROLES.add(new ProviderRole(HBaseKeys.ROLE_REST_GATEWAY, KEY_REST_GATEWAY)); - ROLES.add(new ProviderRole(HBaseKeys.ROLE_THRIFT_GATEWAY, KEY_THRIFT_GATEWAY)); - ROLES.add(new ProviderRole(HBaseKeys.ROLE_THRIFT_GATEWAY, KEY_THRIFT2_GATEWAY)); - } - - - public static List<ProviderRole> getRoles() { - return ROLES; - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hadoop-metrics.properties ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hadoop-metrics.properties b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hadoop-metrics.properties deleted file mode 100644 index fd37d44..0000000 --- a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hadoop-metrics.properties +++ /dev/null @@ -1,85 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# See http://wiki.apache.org/hadoop/GangliaMetrics -# Make sure you know whether you are using ganglia 3.0 or 3.1. -# If 3.1, you will have to patch your hadoop instance with HADOOP-4675 -# And, yes, this file is named hadoop-metrics.properties rather than -# hbase-metrics.properties because we're leveraging the hadoop metrics -# package and hadoop-metrics.properties is an hardcoded-name, at least -# for the moment. -# -# See also http://hadoop.apache.org/hbase/docs/current/metrics.html -# GMETADHOST_IP is the hostname (or) IP address of the server on which the ganglia -# meta daemon (gmetad) service is running - -# Configuration of the "hbase" context for NullContextWithUpdateThread -# NullContextWithUpdateThread is a null context which has a thread calling -# periodically when monitoring is started. This keeps the data sampled -# correctly. -hbase.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread -hbase.period=10 - -# Configuration of the "hbase" context for file -# hbase.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext -# hbase.fileName=/tmp/metrics_hbase.log - -# HBase-specific configuration to reset long-running stats (e.g. compactions) -# If this variable is left out, then the default is no expiration. -hbase.extendedperiod = 3600 - -# Configuration of the "hbase" context for ganglia -# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) -# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext -# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 -# hbase.period=10 -# hbase.servers=GMETADHOST_IP:8649 - -# Configuration of the "jvm" context for null -jvm.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread -jvm.period=10 - -# Configuration of the "jvm" context for file -# jvm.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext -# jvm.fileName=/tmp/metrics_jvm.log - -# Configuration of the "jvm" context for ganglia -# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) -# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext -# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 -# jvm.period=10 -# jvm.servers=GMETADHOST_IP:8649 - -# Configuration of the "rpc" context for null -rpc.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread -rpc.period=10 - -# Configuration of the "rpc" context for file -# rpc.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext -# rpc.fileName=/tmp/metrics_rpc.log - -# Configuration of the "rpc" context for ganglia -# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) -# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext -# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 -# rpc.period=10 -# rpc.servers=GMETADHOST_IP:8649 - -# Configuration of the "rest" context for ganglia -# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) -# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext -# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 -# rest.period=10 -# rest.servers=GMETADHOST_IP:8649 http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-env.sh ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-env.sh b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-env.sh deleted file mode 100644 index 57c9c26..0000000 --- a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-env.sh +++ /dev/null @@ -1,100 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Set environment variables here. - -# This script sets variables multiple times over the course of starting an hbase process, -# so try to keep things idempotent unless you want to take an even deeper look -# into the startup scripts (bin/hbase, etc.) - -# The java implementation to use. Java 1.6 required. -# export JAVA_HOME=/usr/java/jdk1.6.0/ - -# Extra Java CLASSPATH elements. Optional. -# export HBASE_CLASSPATH= - -# The maximum amount of heap to use, in MB. Default is 1000. -# export HBASE_HEAPSIZE=1000 - -# Extra Java runtime options. -# Below are what we set by default. May only work with SUN JVM. -# For more on why as well as other possible settings, -# see http://wiki.apache.org/hadoop/PerformanceTuning -export HBASE_OPTS="-XX:+UseConcMarkSweepGC" - -# Uncomment below to enable java garbage collection logging for the server-side processes -# this enables basic gc logging for the server processes to the .out file -# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS" - -# this enables gc logging using automatic GC log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. Either use this set of options or the one above -# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M $HBASE_GC_OPTS" - -# Uncomment below to enable java garbage collection logging for the client processes in the .out file. -# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS" - -# Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS). -# This applies to both the server and client GC options above -# export HBASE_USE_GC_LOGFILE=true - - -# Uncomment below if you intend to use the EXPERIMENTAL off heap cache. -# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=" -# Set hbase.offheapcache.percentage in hbase-site.xml to a nonzero value. - - -# Uncomment and adjust to enable JMX exporting -# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. -# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html -# -# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" -# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" -# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" -# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" -# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" - -# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. -# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers - -# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default. -# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters - -# Extra ssh options. Empty by default. -# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" - -# Where log files are stored. $HBASE_HOME/logs by default. -# export HBASE_LOG_DIR=${HBASE_HOME}/logs - -# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers -# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070" -# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071" -# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072" -# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073" - -# A string representing this instance of hbase. $USER by default. -# export HBASE_IDENT_STRING=$USER - -# The scheduling priority for daemon processes. See 'man nice'. -# export HBASE_NICENESS=10 - -# The directory where pid files are stored. /tmp by default. -# export HBASE_PID_DIR=/var/hadoop/pids - -# Seconds to sleep between slave commands. Unset by default. This -# can be useful in large clusters, where, e.g., slave rsyncs can -# otherwise arrive faster than the master can service them. -# export HBASE_SLAVE_SLEEP=0.1 - -# Tell HBase whether it should manage it's own instance of Zookeeper or not. -# export HBASE_MANAGES_ZK=true http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-policy.xml ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-policy.xml b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-policy.xml deleted file mode 100644 index e45f23c..0000000 --- a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-policy.xml +++ /dev/null @@ -1,53 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> - -<configuration> - <property> - <name>security.client.protocol.acl</name> - <value>*</value> - <description>ACL for HRegionInterface protocol implementations (ie. - clients talking to HRegionServers) - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed.</description> - </property> - - <property> - <name>security.admin.protocol.acl</name> - <value>*</value> - <description>ACL for HMasterInterface protocol implementation (ie. - clients talking to HMaster for admin operations). - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed.</description> - </property> - - <property> - <name>security.masterregion.protocol.acl</name> - <value>*</value> - <description>ACL for HMasterRegionInterface protocol implementations - (for HRegionServers communicating with HMaster) - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed.</description> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-site.xml ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-site.xml b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-site.xml deleted file mode 100644 index 2b21659..0000000 --- a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/hbase-site.xml +++ /dev/null @@ -1,51 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - - - <!-- - This is a template HBase site file that - does not include all the properties - required for a valid configuration - - the remainder are injected during - conversion from a template to - actual file - --> -<configuration> - <property> - <name>slider.template.origin</name> - <value>resource</value> - </property> - <property> - <name>hbase.cluster.distributed</name> - <value>true</value> - </property> - <property> - <name>hbase.regionserver.info.port.auto</name> - <value>true</value> - </property> - <property> - <name>hbase.tmp.dir</name> - <value>./hbase-tmp</value> - </property> - <property> - <name>hbase.regionserver.hlog.tolerable.lowreplication</name> - <value>1</value> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/log4j.properties ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/log4j.properties b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/log4j.properties deleted file mode 100644 index 3ea8b71..0000000 --- a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/log4j.properties +++ /dev/null @@ -1,91 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Define some default values that can be overridden by system properties -hbase.root.logger=INFO,DRFA -hbase.security.logger=INFO,DRFA -hbase.log.dir=. -hbase.log.file=hbase.log - -# Define the root logger to the system property "hbase.root.logger". -log4j.rootLogger=${hbase.root.logger} - -# Logging Threshold -log4j.threshold=ALL - -# -# Daily Rolling File Appender -# -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} - -# Rollver at midnight -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd - -# 30-day backup -#log4j.appender.DRFA.MaxBackupIndex=30 -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n - -# Debugging Pattern format -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n - -# -# Security audit appender -# -hbase.security.log.file=SecurityAuth.audit -log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFAS.File=${hbase.log.dir}/${hbase.security.log.file} -log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout -log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -log4j.category.SecurityLogger=${hbase.security.logger} -log4j.additivity.SecurityLogger=false -#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE - -# -# Null Appender -# -log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender - -# -# console -# Add "console" to rootlogger above if you want to use this -# -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n - -# Custom Logging levels - -log4j.logger.org.apache.zookeeper=INFO -log4j.logger.org.apache.zookeeper.ClientCnxn=ERROR -#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG -log4j.logger.org.apache.hadoop.hbase=DEBUG -# Make these two classes INFO-level. Make them DEBUG to see more zk debug. -log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO -log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO -#log4j.logger.org.apache.hadoop.dfs=DEBUG -# Set this class to log INFO only otherwise its OTT - -# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output) -#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG - -# Uncomment the below if you want to remove logging of client region caching' -# and scan of .META. messages -# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO -# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/regionservers ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/regionservers b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/conf/regionservers deleted file mode 100644 index e69de29..0000000 http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/appconf.json ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/appconf.json b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/appconf.json deleted file mode 100644 index 5320354..0000000 --- a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/appconf.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "schema": "http://example.org/specification/v2.0.0";, - - "metadata": { - - - }, - - "global": { - "env.MALLOC_ARENA_MAX": "4", - "site.hbase.master.startup.retainassign": "true", - "site.hbase.cluster.distributed": "true", - "site.hbase.regionserver.info.port.auto": "true", - "site.hbase.tmp.dir": "./hbase-tmp", - "site.hbase.regionserver.hlog.tolerable.lowreplication": "1", - "site.hbase.master.port": "0", - "site.hbase.master.info.port": "0", - "site.hbase.regionserver.port": "0", - "site.hbase.regionserver.info.port": "0", - "jvm.heapsize": "256M" - }, - - "components": { - "master": { - }, - "worker": { - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/resources.json ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/resources.json b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/resources.json deleted file mode 100644 index 9a0db91..0000000 --- a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/instance/resources.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "schema": "http://example.org/specification/v2.0.0";, - - "metadata": { - - }, - - "global": { - - }, - - "components": { - "master": { - "yarn.component.instances": "1", - "yarn.vcores": "1", - "yarn.memory": "1024" - }, - "worker": { - "yarn.component.instances": "2", - "yarn.vcores": "1", - "yarn.memory": "768" - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/role-hbase-master.xml ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/role-hbase-master.xml b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/role-hbase-master.xml deleted file mode 100644 index 005e56d..0000000 --- a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/role-hbase-master.xml +++ /dev/null @@ -1,59 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - - <!-- - These are the default cluster options for a Slider cluster - --> -<configuration> - <property> - <name>role.name</name> - <value>master</value> - </property> - - <property> - <name>role.instances</name> - <value>1</value> - </property> - - <property> - <name>yarn.vcores</name> - <value>1</value> - </property> - - <property> - <name>yarn.memory</name> - <value>1024</value> - </property> - - <property> - <name>jvm.heapsize</name> - <value>512M</value> - </property> - - <property> - <name>app.infoport</name> - <value>0</value> - </property> - <property> - <name>env.MALLOC_ARENA_MAX</name> - <value>4</value> - </property> - -</configuration> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d405543/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/role-hbase-worker.xml ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/role-hbase-worker.xml b/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/role-hbase-worker.xml deleted file mode 100644 index 39ba99e..0000000 --- a/slider-providers/hbase/slider-hbase-provider/src/main/resources/org/apache/slider/providers/hbase/role-hbase-worker.xml +++ /dev/null @@ -1,60 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - - <!-- - These are the default cluster options for a Slider cluster - --> -<configuration> - <property> - <name>role.name</name> - <value>worker</value> - </property> - - <property> - <name>role.instances</name> - <value>2</value> - </property> - - <property> - <name>yarn.vcores</name> - <value>1</value> - </property> - - <property> - <name>yarn.memory</name> - <value>768</value> - </property> - - <property> - <name>jvm.heapsize</name> - <value>512M</value> - </property> - - <property> - <name>app.infoport</name> - <value>0</value> - </property> - - <property> - <name>env.MALLOC_ARENA_MAX</name> - <value>4</value> - </property> - -</configuration>
