Repository: bigtop Updated Branches: refs/heads/master 738ff0746 -> ebcdedf3d
BIGTOP-2727. fix deployment tasks Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/ebcdedf3 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/ebcdedf3 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/ebcdedf3 Branch: refs/heads/master Commit: ebcdedf3dd24df38d8ed7b7ee547b11d8268db4e Parents: 738ff07 Author: Roman Shaposhnik <[email protected]> Authored: Tue Mar 28 16:41:49 2017 -0700 Committer: Roman Shaposhnik <[email protected]> Committed: Tue Mar 28 16:43:39 2017 -0700 ---------------------------------------------------------------------- bigtop-tests/build.gradle | 5 + bigtop-tests/test-artifacts/fatjar/pom.xml | 11 +- .../hbase/system/TestMoveRootMetaRegions.java | 150 ------------------ pom.xml | 151 ++++++------------- release.gradle | 42 +++--- 5 files changed, 80 insertions(+), 279 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/ebcdedf3/bigtop-tests/build.gradle ---------------------------------------------------------------------- diff --git a/bigtop-tests/build.gradle b/bigtop-tests/build.gradle index 2386d7e..f950fc0 100644 --- a/bigtop-tests/build.gradle +++ b/bigtop-tests/build.gradle @@ -57,6 +57,11 @@ distributions { } } +tasks.withType(Tar) { + compression = Compression.GZIP + extension = "tar.gz" +} + distZip.dependsOn clean distTar.dependsOn clean http://git-wip-us.apache.org/repos/asf/bigtop/blob/ebcdedf3/bigtop-tests/test-artifacts/fatjar/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/fatjar/pom.xml b/bigtop-tests/test-artifacts/fatjar/pom.xml index ca01eda..ba0a942 100644 --- a/bigtop-tests/test-artifacts/fatjar/pom.xml +++ b/bigtop-tests/test-artifacts/fatjar/pom.xml @@ -128,19 +128,16 @@ <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-assembly-plugin</artifactId> + <artifactId>maven-shade-plugin</artifactId> + <version>3.0.0</version> <configuration> - <appendAssemblyId>false</appendAssemblyId> - <descriptorRefs> - <descriptorRef>jar-with-dependencies</descriptorRef> - </descriptorRefs> + <!-- put your configurations here --> </configuration> <executions> <execution> - <id>make-assembly</id> <phase>package</phase> <goals> - <goal>assembly</goal> + <goal>shade</goal> </goals> </execution> </executions> http://git-wip-us.apache.org/repos/asf/bigtop/blob/ebcdedf3/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestMoveRootMetaRegions.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestMoveRootMetaRegions.java b/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestMoveRootMetaRegions.java deleted file mode 100644 index 2132a43..0000000 --- a/bigtop-tests/test-artifacts/hbase/src/main/groovy/org/apache/bigtop/itest/hbase/system/TestMoveRootMetaRegions.java +++ /dev/null @@ -1,150 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.bigtop.itest.hbase.system; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.not; - -import org.apache.bigtop.itest.shell.Shell; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.catalog.CatalogTracker; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * This test contains 3 tests: - * <p/> - * 1) Unload the region server hosting -ROOT-. Check that -ROOT- region - * is accessible after a configurable period of time. - * <p/> - * 2) The test above for the .META. region. - * <p/> - * 3) Stop the region server(s) hosting the -ROOT- and .META. tables - * and verify the regions are moved to other region server(s). - * <p/> - * This test does not restart the stopped region server, so users will - * have to manually restart the region server. - * <p/> - * The third test is designed for clusters with more than two region servers. - * <p/> - * Optional arguments: - * -Droot.timeout.ms=<milliseconds to wait while trying to find -ROOT-> - * -Dmeta.timeout.ms=<milliseconds to wait while trying to find .META.> - * -Dwait.after.move.ms=<milliseconds to wait after moving -ROOT- or .META.> - */ - -public class TestMoveRootMetaRegions { - private static final String HBASE_HOME = System.getenv("HBASE_HOME"); - - static { - Assert.assertNotNull("HBASE_HOME has to be set to run this test", HBASE_HOME); - } - - private static Shell sh = new Shell("/bin/bash -s"); - - private static String load_regionserver = - "$HBASE_HOME/bin/hbase org.jruby.Main $HBASE_HOME/bin/region_mover.rb load "; - private static String unload_regionserver = - "$HBASE_HOME/bin/hbase org.jruby.Main $HBASE_HOME/bin/region_mover.rb unload "; - - private static long meta_timeout_ms; - private static long root_timeout_ms; - private static long wait_after_move_ms; - - private static Configuration conf; - private static HBaseAdmin admin; - private static CatalogTracker ct; - - private static String meta_table = - Bytes.toStringBinary(TableName.META_TABLE_NAME.getName()); - - @BeforeClass - public static void setUp() throws Exception { - // Default timeout is 3 minutes. - root_timeout_ms = - Integer.parseInt(System.getProperty("root.timeout.ms", "180000")); - meta_timeout_ms = - Integer.parseInt(System.getProperty("meta.timeout.ms", "180000")); - // Default to 20 seconds. - wait_after_move_ms = - Integer.parseInt(System.getProperty("wait.after.move.ms", "20000")); - - conf = HBaseConfiguration.create(); - admin = new HBaseAdmin(conf); - - ct = new CatalogTracker(admin.getConfiguration()); - ct.start(); - - Assert.assertTrue(admin.tableExists(HConstants.META_TABLE_NAME)); - Assert.assertTrue(admin.isTableEnabled(HConstants.META_TABLE_NAME)); - } - - @AfterClass - public static void tearDown() { - // Cleanup - ct.stop(); - } - - public static ServerName getMetaAddress() throws Exception { - return ct.waitForMeta(meta_timeout_ms); - } - - @Test - public void unloadMetaRegionServer() throws Exception { - ServerName meta_address = getMetaAddress(); - String cmd = unload_regionserver + meta_address.getHostname(); - System.out.println("Unloading the region server hosting " + meta_table); - System.out.println(cmd); - sh.exec(cmd); - - Thread.sleep(wait_after_move_ms); - getMetaAddress(); - - cmd = load_regionserver + meta_address.getHostname(); - System.out.println("Reloading the region server"); - sh.exec(cmd); - Thread.sleep(wait_after_move_ms); - } - - @Test - public void testStopRootMetaRegionServers() throws Exception { - ServerName meta_address = getMetaAddress(); - - boolean same_server = false; - - System.out.println(meta_table + " server address: " + meta_address); - - System.out.println("Stopping region server(s)"); - admin.stopRegionServer(meta_address.getHostAndPort()); - - Thread.sleep(wait_after_move_ms); - - ServerName new_meta_address = getMetaAddress(); - - System.out.println(meta_table + " server address: " + new_meta_address); - Assert.assertThat(meta_address, not(equalTo(new_meta_address))); - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/ebcdedf3/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index f128314..5c5de30 100644 --- a/pom.xml +++ b/pom.xml @@ -39,19 +39,19 @@ <inceptionYear>2011</inceptionYear> <properties> - <hadoop.version>2.7.2</hadoop.version> + <hadoop.version>2.7.3</hadoop.version> <!--An awful hack for BIGTOP-1429--> <hbase.version>1.1.3</hbase.version> - <pig.version>0.12.1</pig.version> - <pig-smoke.version>0.12.1</pig-smoke.version> - <sqoop.version>1.4.5</sqoop.version> + <pig.version>0.15.0</pig.version> + <pig-smoke.version>0.15.0</pig-smoke.version> + <sqoop.version>1.4.6</sqoop.version> <sqoop2.version>1.99.4</sqoop2.version> - <zookeeper.version>3.4.5</zookeeper.version> - <giraph.version>1.0.0</giraph.version> - <solr.version>4.6.0</solr.version> - <spark.version>1.3.1</spark.version> - <kafka.version>0.8.1.1</kafka.version> - <phoenix.version>4.7.0</phoenix.version> + <zookeeper.version>3.4.6</zookeeper.version> + <giraph.version>1.1.0</giraph.version> + <solr.version>4.10.4</solr.version> + <spark.version>2.1.0</spark.version> + <kafka.version>0.10.1.1</kafka.version> + <phoenix.version>4.9.0-HBase-1.1</phoenix.version> <kite.version>1.1.0</kite.version> <spark-smoke.version>${project.version}</spark-smoke.version> @@ -290,86 +290,6 @@ <pluginManagement> <plugins> <plugin> - <groupId>org.apache.rat</groupId> - <artifactId>apache-rat-plugin</artifactId> - <version>0.7</version> - <configuration> - <excludes> - <exclude>.git/**</exclude> - <exclude>.idea/**</exclude> - <exclude>**/.gitignore</exclude> - <!-- Test data with rigid structure and/or binary --> - <exclude>bigtop-tests/test-artifacts/**/resources/**</exclude> - <exclude>bigtop-tests/smoke-tests/alluxio/datafile</exclude> - <exclude>bigtop-tests/smoke-tests/hive/passwd.ql</exclude> - <exclude>bigtop-tests/smoke-tests/kite/sandwiches.csv</exclude> - <exclude>bigtop-tests/smoke-tests/phoenix/*.csv</exclude> - <exclude>bigtop-tests/smoke-tests/phoenix/*.sql</exclude> - <exclude>bigtop-tests/smoke-tests/ignite-hadoop/*.data</exclude> - <exclude>bigtop-tests/smoke-tests/tajo/table1/*.csv</exclude> - <exclude>**/target/**</exclude> - <exclude>**/build/**</exclude> - <exclude>**/.gradle/**</exclude> - <exclude>**/*.iml</exclude> - <exclude>.gradle</exclude> - <exclude>output/**</exclude> - <exclude>MAINTAINERS.txt</exclude> - <exclude>buildSrc/build/**</exclude> - <exclude>bigtop-data-generators/**/input_data/**</exclude> - <exclude>docker/**/*.yaml</exclude> - <exclude>bigtop-deploy/puppet/**/*.yaml</exclude> - <exclude>bigtop-deploy/puppet/modules/hadoop/**/id_hdfsuser*</exclude> - <exclude>bigtop-deploy/puppet/modules/bigtop-util/Gemfile.lock</exclude> - <exclude>bigtop-deploy/puppet/modules/zeppelin/templates/interpreter.json</exclude> - <!-- Debian package files that have a rigid structure (* matches package name) --> - <exclude>bigtop-packages/src/deb/*/*.dirs</exclude> - <exclude>bigtop-packages/src/deb/*/dirs</exclude> - <exclude>bigtop-packages/src/deb/*/*.install</exclude> - <exclude>bigtop-packages/src/deb/*/*.include</exclude> - <exclude>bigtop-packages/src/deb/**/include-binaries</exclude> - <exclude>bigtop-packages/src/deb/**/format</exclude> - <exclude>bigtop-packages/src/deb/*/*.manpages</exclude> - <exclude>bigtop-packages/src/deb/*/*.docs</exclude> - <exclude>bigtop-packages/src/deb/*/docs</exclude> - <exclude>bigtop-packages/src/deb/*/compat</exclude> - <exclude>bigtop-packages/src/deb/*/source/format</exclude> - <exclude>bigtop-packages/src/deb/*/*.lintian-overrides</exclude> - <exclude>bigtop-packages/src/deb/*/shlibs.local</exclude> - <exclude>bigtop-packages/src/**/*.diff</exclude> - <exclude>bigtop-packages/src/common/*/*.json</exclude> - <exclude>bigtop-packages/src/common/**/*.default</exclude> - <exclude>bigtop-repos/apt/distributions</exclude> - <!-- Juju charm files with rigid structure --> - <exclude>bigtop-packages/src/charm/**/wheelhouse.txt</exclude> - <exclude>bigtop-packages/src/charm/**/*.json</exclude> - <exclude>bigtop-packages/src/charm/**/*.svg</exclude> - <exclude>bigtop-packages/src/charm/**/*.yaml</exclude> - <exclude>bigtop-deploy/juju/**/*.yaml</exclude> - <exclude>bigtop-tests/cloud-weather-report/**/*.yaml</exclude> - <!-- Miscelaneous individual files --> - <exclude>src/site/resources/bigtop.rdf</exclude> - <exclude>src/site/resources/images/bigtop-logo.ai</exclude> - <!--Excluding a file with FreeBSD Documentation License--> - <exclude>src/site/resources/css/freebsd_docbook.css</exclude> - <exclude>test/MANIFEST.txt</exclude> - <exclude> - bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg - </exclude> - <exclude> - bigtop-deploy/puppet/modules/hadoop/templates/container-executor.cfg - </exclude> - <exclude>bigtop-deploy/puppet/README.md</exclude> - <exclude>bigtop-deploy/puppet/modules/qfs/README.md</exclude> - <exclude>bigtop-deploy/puppet/config/site.csv.example</exclude> - <exclude>dl/**</exclude> - <exclude>bigtop-deploy/puppet/modules/gpdb/templates/**</exclude> - <exclude>provisioner/docker/config/hiera.yaml</exclude> - <exclude>provisioner/docker/config/hieradata/PLACEHOLDER</exclude> - <exclude>provisioner/docker/config/hosts</exclude> - </excludes> - </configuration> - </plugin> - <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-pdf-plugin</artifactId> <version>1.2</version> @@ -399,6 +319,11 @@ <artifactId>maven-scm-publish-plugin</artifactId> <version>1.0</version> </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>3.0.0</version> + </plugin> </plugins> </pluginManagement> <plugins> @@ -488,6 +413,40 @@ <profiles> <profile> + <id>attach-smokes</id> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <executions> + <execution> + <id>attach-artifacts</id> + <phase>package</phase> + <goals> + <goal>attach-artifact</goal> + </goals> + <configuration> + <artifacts> + <artifact> + <file>bigtop-tests/build/distributions/bigtop-smokes-suite-${project.version}.tar.gz</file> + <type>tar.gz</type> + <classifier>testsuite</classifier> + </artifact> + <artifact> + <file>bigtop-tests/build/distributions/bigtop-smokes-suite-${project.version}.zip</file> + <type>zip</type> + <classifier>testsuite</classifier> + </artifact> + </artifacts> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> <id>release</id> <build> <plugins> @@ -509,18 +468,6 @@ </executions> </plugin> <plugin> - <groupId>org.apache.rat</groupId> - <artifactId>apache-rat-plugin</artifactId> - <executions> - <execution> - <phase>verify</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-gpg-plugin</artifactId> <executions> http://git-wip-us.apache.org/repos/asf/bigtop/blob/ebcdedf3/release.gradle ---------------------------------------------------------------------- diff --git a/release.gradle b/release.gradle index 7d1e5f5..6b9afca 100644 --- a/release.gradle +++ b/release.gradle @@ -16,45 +16,47 @@ * limitations under the License. * */ - apply plugin: 'groovy' def final mvnProfile="release" -task dosite (type: Exec) << { +task dosite (type: Exec) { workingDir "." - commandLine 'mvn clean site'.split(" ") + commandLine "mvn clean site".split(" ") } -task doassembly (type: Exec) << { +task doassembly (type: Exec) { workingDir "." - commandLine 'mvn -P$mvnProfile package assembly:assembly'.split(" ") + commandLine "mvn -P$mvnProfile package assembly:assembly".split(" ") } -task deployTop (type: Exec) << { +task deployTop (type: Exec, dependsOn: ["bigtop-tests:assembleDist"]) { workingDir "." - commandLine 'mvn -P$mvnProfile deploy -f pom.xml'.split(" ") + commandLine "mvn -P$mvnProfile -Pattach-smokes deploy -f pom.xml".split(" ") + standardInput System.in } -task deployITest (type: Exec) << { +task deployITest (type: Exec) { workingDir "." - commandLine 'mvn -P$mvnProfile deploy -f bigtop-test-framework/pom.xml'.split(" ") + commandLine "mvn -P$mvnProfile -DskipTests deploy -f bigtop-test-framework/pom.xml".split(" ") + standardInput System.in } -task deployTestArtifacts (type: Exec) << { +task deployTestArtifacts (type: Exec) { workingDir "." - commandLine 'mvn -P$mvnProfile deploy -f bigtop-tests/test-artifacts/pom.xml'.split(" ") + commandLine "mvn -P$mvnProfile deploy -f bigtop-tests/test-artifacts/pom.xml".split(" ") + standardInput System.in } -task deployTestExecutors (type: Exec) << { +task deployTestExecutors (type: Exec) { workingDir "." - commandLine 'mvn -P$mvnProfile deploy -f bigtop-tests/test-execution/pom.xml'.split(" ") + commandLine "mvn -P$mvnProfile deploy -f bigtop-tests/test-execution/pom.xml".split(" ") + environment HADOOP_CONF_DIR: "", HADOOP_HOME: "" + standardInput System.in } -task deployAll (type: Exec, - dependsOn: [deployTestExecutors, deployTestArtifacts, - deployITest, deployTop]) << { - workingDir "." - commandLine 'mvn deploy -Prelease'.split(" ") +task deployAll (dependsOn: [deployTestExecutors, deployTestArtifacts, + deployITest, deployTop]) << { + println "Done!" } task dorelease(description: 'Convenient wrapper for existing release tasks', - dependsOn: [doassembly, dosite, deployAll]) << { + dependsOn: [dosite, deployAll]) << { } task setversion << { @@ -67,4 +69,4 @@ task setversion << { pomFile.write(pomFile.text.replaceAll(version, nextversion)) } println "Done! Please inspect changes to make sure everything is in order." -} \ No newline at end of file +}
