Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/pom.xml URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/pom.xml?rev=1196868&r1=1196867&r2=1196868&view=diff ============================================================================== --- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/pom.xml (original) +++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/pom.xml Wed Nov 2 23:40:24 2011 @@ -49,11 +49,33 @@ <artifactId>ant-junit</artifactId> <version>1.8.2</version> </dependency> - - <dependency> - <groupId>org.apache.bigtop.itest</groupId> - <artifactId>hadoop-smoke</artifactId> - <version>0.2.0-incubating-SNAPSHOT</version> - </dependency> </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.codehaus.groovy.maven</groupId> + <artifactId>gmaven-plugin</artifactId> + <version>1.0</version> + <executions> + <execution> + <id>check-manifest</id> + <phase>process-test-classes</phase> + <goals> + <goal>execute</goal> + </goals> + <configuration> + <source><![CDATA[ + ["yum", "zypper", "apt", "urpmi"].each { + def a = new XmlSlurper().parseText(new File("${project.build.directory}/classes/package_data_${it}.xml").text); + } + println "Manifest XML validated"; + ]]> + </source> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> </project>
Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/CDHServices.groovy URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/CDHServices.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff ============================================================================== --- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/CDHServices.groovy (original) +++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/CDHServices.groovy Wed Nov 2 23:40:24 2011 @@ -21,47 +21,65 @@ package org.apache.bigtop.itest.packages import org.apache.bigtop.itest.pmanager.PackageManager class CDHServices { + static final List serviceDaemonUserNames = [ "flume", "hbase" , "hdfs" , "hue" , "mapred" , "oozie" , "sqoop" , + "zookeeper", "hadoop"]; + static final Map components = [ - HDFS : [ services : [ "hadoop-0.20-namenode", "hadoop-0.20-datanode", - "hadoop-0.20-secondarynamenode" ], + HDFS : [ services : [ "hadoop-namenode", "hadoop-datanode", + "hadoop-secondarynamenode" ], verifier : new StateVerifierHDFS(), + killIDs : [ "hdfs" ], ], - mapreduce : [ services : [ "hadoop-0.20-namenode", "hadoop-0.20-datanode", - "hadoop-0.20-jobtracker", "hadoop-0.20-tasktracker" ], + mapreduce : [ services : [ "hadoop-namenode", "hadoop-datanode", + "hadoop-jobtracker", "hadoop-tasktracker" ], + killIDs : [ "hdfs", "mapred" ], verifier : new StateVerifierMapreduce(), ], - HBase : [ services : [ "hadoop-0.20-namenode", "hadoop-0.20-datanode", + hive : [ services : [ "hadoop-namenode", "hadoop-datanode", + "hadoop-jobtracker", "hadoop-tasktracker" ], + killIDs : [ "hdfs", "mapred" ], + verifier : new StateVerifierHive(), + ], + HBase : [ services : [ "hadoop-namenode", "hadoop-datanode", "hadoop-hbase-master" ], + killIDs : [ "hdfs", "hbase" ], verifier : new StateVerifierHBase(), ], - zookeeper : [ services : [ "${PackageManager.getPackageManager().type == 'apt' ? 'hadoop-zookeeper-server' : 'hadoop-zookeeper'}" ], + zookeeper : [ services : [ "hadoop-zookeeper" ], verifier : new StateVerifierZookeeper(), + killIDs : [ "zookeeper" ], ], - oozie : [ services : [ "hadoop-0.20-namenode", "hadoop-0.20-datanode", "hadoop-0.20-jobtracker", "hadoop-0.20-tasktracker", + oozie : [ services : [ "hadoop-namenode", "hadoop-datanode", "hadoop-jobtracker", "hadoop-tasktracker", "oozie" ], + killIDs : [ "hdfs", "mapred", "oozie" ], verifier : new StateVerifierOozie(), ], - flume : [ services : [ "hadoop-0.20-namenode", "hadoop-0.20-datanode", + flume : [ services : [ "hadoop-namenode", "hadoop-datanode", "flume-master", "flume-node" ], + killIDs : [ "hdfs", "flume" ], verifier : new StateVerifierFlume(), ], - sqoop : [ services : [ "hadoop-0.20-namenode", "hadoop-0.20-datanode", + sqoop : [ services : [ "hadoop-namenode", "hadoop-datanode", "sqoop-metastore" ], + killIDs : [ "hdfs", "sqoop" ], verifier : new StateVerifierSqoop(), ], - hue : [ services : [ "hadoop-0.20-namenode", "hadoop-0.20-datanode", "hadoop-0.20-jobtracker", "hadoop-0.20-tasktracker", + hue : [ services : [ "hadoop-namenode", "hadoop-datanode", "hadoop-jobtracker", "hadoop-tasktracker", "hue" ], + killIDs : [ "hdfs", "mapred", "hue" ], verifier : new StateVerifierHue(), ], ]; static final Map<String, List<String>> release2services = [ - "2" : [ "HDFS", "mapreduce" ], - "3b2" : [ "HDFS", "mapreduce", "HBase", "zookeeper", "oozie", "flume", "hue" ], - "3b3" : [ "HDFS", "mapreduce", "HBase", "zookeeper", "oozie", "flume", "sqoop", "hue" ], - "3b4" : [ "HDFS", "mapreduce", "HBase", "zookeeper", "oozie", "flume", "sqoop", "hue" ], - "3u0" : [ "HDFS", "mapreduce", "HBase", "zookeeper", "oozie", "flume", "sqoop", "hue" ], - "3" : [ "HDFS", "mapreduce", "HBase", "zookeeper", "oozie", "flume", "sqoop", "hue" ], + "bigtop" : [ "HDFS", "mapreduce", "hive", "HBase", "zookeeper", "oozie", "flume" ], + "2" : [ "HDFS", "mapreduce", "hive" ], + "3b2" : [ "HDFS", "mapreduce", "hive", "HBase", "zookeeper", "oozie", "flume", "hue" ], + "3b3" : [ "HDFS", "mapreduce", "hive", "HBase", "zookeeper", "oozie", "flume", "sqoop", "hue" ], + "3b4" : [ "HDFS", "mapreduce", "hive", "HBase", "zookeeper", "oozie", "flume", "sqoop", "hue" ], + "3u0" : [ "HDFS", "mapreduce", "hive", "HBase", "zookeeper", "oozie", "flume", "sqoop", "hue" ], + "3u1" : [ "HDFS", "mapreduce", "hive", "HBase", "zookeeper", "oozie", "flume", "sqoop", "hue" ], + "3" : [ "HDFS", "mapreduce", "hive", "HBase", "zookeeper", "oozie", "flume" ], ]; public static Map getServices(String release) { @@ -69,8 +87,10 @@ class CDHServices { List<String> services; if ((release =~ /\.\./).find()) { - services = release2services[release.replaceAll(/^.*\.\./, "")].clone(); - release2services[release.replaceAll(/\.\..*$/, "")].each { + String release_from = release.replaceAll(/\.\..*$/, ""); + release = release.replaceAll(/^.*\.\./, ""); + services = release2services[release].clone(); + release2services[release_from].each { services.remove(it); } } else { @@ -78,7 +98,16 @@ class CDHServices { } services.each { - res[it] = components[it]; + // zookeeper is a very messy case of naming :-( + if (it == "zookeeper" && + (PackageManager.getPackageManager().type == 'apt' || + release == "3" || release == "3u1" || release == "bigtop")) { + res[it] = [ services : [ "hadoop-zookeeper-server" ], + verifier : new StateVerifierZookeeper(), + ]; + } else { + res[it] = components[it]; + } } return res; } Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/DeployCDH.groovy URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/DeployCDH.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff ============================================================================== --- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/DeployCDH.groovy (original) +++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/DeployCDH.groovy Wed Nov 2 23:40:24 2011 @@ -58,13 +58,16 @@ class DeployCDH { Map<String, LinkedHashMap<String, Collection>> distPackages = [ "2" : [ "apt" : cdh2 + aptPkg, "yum" : cdh2 + yumPkg + [ "hadoop-0.20-docs", "hadoop-hive-webinterface" ], + "zypper" : [], // "cloudera-desktop", "cloudera-desktop-plugins", ], "3b2" : [ "apt" : cdh3b2 + aptPkg, "yum" : cdh3b2 + yumPkg + [ "hadoop-0.20-docs", "hadoop-hive-webinterface" ], + "zypper" : [], ], "3b3" : [ "apt" : cdh3b3 + aptPkg + [ "hadoop-hbase-doc", ], "yum" : cdh3b3 + yumPkg + [ "hadoop-0.20-docs", "hadoop-hive-webinterface" ], + "zypper" : [], ], "3b4" : [ "apt" : cdh3b3 + aptPkg + [ "hadoop-hbase-doc", ], "yum" : cdh3b3 + yumPkg + [ "hadoop-0.20-doc", "hadoop-hive-webinterface" ], @@ -78,6 +81,10 @@ class DeployCDH { "yum" : cdh3b3 + yumPkg + [ "hadoop-0.20-doc" ], "zypper" : cdh3b3 + zypperPkg, ], + "3u1" : [ "apt" : cdh3b3 + aptPkg + [ "hadoop-hbase-doc", ], + "yum" : cdh3b3 + yumPkg + [ "hadoop-0.20-doc" ], + "zypper" : cdh3b3 + zypperPkg, + ], ]; @Rule @@ -94,10 +101,22 @@ class DeployCDH { checkThat("failed to add repository for pre-upgrade CDH deployment", oldRepo.getPm().refresh(), equalTo(0)); + // Lets try to remove existing packages -- just in case + List stalePkgs = []; + distPackages.each { key, value -> stalePkgs.addAll(value[pm.type]); } + (stalePkgs as Set).each { + PackageInstance pkg = PackageInstance.getPackageInstance(pm, it); + pkg.remove(); + } + packages.each { PackageInstance pkg = PackageInstance.getPackageInstance(pm, it); - checkThat("failed to install required package ${pkg.getName()}", - pkg.install(), equalTo(0)); + // Some packages get installed as requirement for others that we install. + // We don't want to install them for a second time. + if (!pkg.isInstalled()) { + checkThat("failed to install required package ${pkg.getName()}", + pkg.install(), equalTo(0)); + } pkg.getServices().each { it.value.stop(); } } Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/PackageTestCommon.groovy URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/PackageTestCommon.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff ============================================================================== --- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/PackageTestCommon.groovy (original) +++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/PackageTestCommon.groovy Wed Nov 2 23:40:24 2011 @@ -30,6 +30,7 @@ import org.hamcrest.Matcher import static org.apache.bigtop.itest.packagesmoke.PackageTestMatchers.MapHasKeys.hasSameKeys import org.apache.bigtop.itest.shell.Shell +import groovy.xml.MarkupBuilder class PackageTestCommon { static public PackageManager pm; @@ -48,7 +49,7 @@ class PackageTestCommon { } String formatDescription(String description, String summary) { - return ((summary ?: "") + ' ' + description).replaceAll(/\s+/,' ').trim(); + return ((summary ?: "") + ' ' + description).replaceAll(/\s+/,' ').replaceAll(/\s\.\s/,' ').replaceAll(/\s\.$/,' ').trim(); } private void checkMetadata(PackageInstance pkg, Map expected_metadata) { @@ -96,7 +97,13 @@ class PackageTestCommon { } public void checkPulledDeps(Map expected_deps) { - Map pkgDeps = pkg.getDeps(); + Map pkgDeps = [:]; + + pkg.getDeps().each { k, v -> + if (!(k =~ /\.so\.[0-9]/).find()) { + pkgDeps[k] = v; + } + } checkThat("a set of dependencies of package $name is different from what was expected", pkgDeps, hasSameKeys(expected_deps)); @@ -367,14 +374,23 @@ class PackageTestCommon { problemFiles, equalTo([])); // a bit of debug output - pkg.getFiles().each { - Map meta = fileMeta[it] ?: [:]; - String target = meta.target ? " target=\"${meta.target}\"" : ""; - String tip = configs[it] ? "config" : (docs[it] ? "doc " : "file"); - println "\n::: ${name} <${tip} name=\"${it}\" owners=\"${meta.owners}\" perm=\"${meta.perm}\"" + - " user=\"${meta.user}\" group=\"${meta.group}\"${target}/>"; + def newManifest = new MarkupBuilder(new FileWriter("${pkg.name}_manifest.xml")); + + newManifest.content() { + fileMeta = getLsMetadata(pkg.getFiles()); + + pkg.getFiles().each { + Map meta = fileMeta[it] ?: [:]; + String node = configs[it] ? "config" : (docs[it] ? "doc " : "file"); + int owners = meta.owners ?: -1; + + if (meta.target) { + "$node"(name : it, owners : owners, perm : meta.perm, user : meta.user, group : meta.group, target : meta.target); + } else { + "$node"(name : it, owners : owners, perm : meta.perm, user : meta.user, group : meta.group); + } + } } - // println ""; } public void checkComplimentary32bitInstall() { Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/PackageTestRepoMgr.groovy URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/PackageTestRepoMgr.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff ============================================================================== --- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/PackageTestRepoMgr.groovy (original) +++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/PackageTestRepoMgr.groovy Wed Nov 2 23:40:24 2011 @@ -50,7 +50,7 @@ class PackageTestRepoMgr { public parseRepoSpec(String prefix) { cdhRepoHost = System.getProperty("${prefix}.host", "nightly.cloudera.com"); - cdhRepoVersion = System.getProperty("${prefix}.version", "3"); + cdhRepoVersion = System.getProperty("${prefix}.version", "bigtop"); Map cdhKeys = [ yum : "http://${cdhRepoHost}/redhat/cdh/RPM-GPG-KEY-cloudera";, zypper : null, @@ -69,6 +69,7 @@ class PackageTestRepoMgr { public boolean addRepo() { repoName = "cloudera-cdh${cdhRepoVersion}"; + pm.cleanup(); try { String repoText = cdhRepoFileURL.toURL().text; if (pm.addBinRepo(repoName, repoText)) { @@ -80,10 +81,11 @@ class PackageTestRepoMgr { return false; } } + pm.refresh(); return true; } public boolean removeRepo() { return (pm.removeBinRepo(repoName) == 0); } -} \ No newline at end of file +} Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierFlume.groovy URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierFlume.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff ============================================================================== --- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierFlume.groovy (original) +++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierFlume.groovy Wed Nov 2 23:40:24 2011 @@ -28,6 +28,7 @@ class StateVerifierFlume extends StateVe public void createState() { String node; + sleep(120001); shFlume.exec("connect localhost", "getnodestatus", "quit\n"); @@ -35,14 +36,15 @@ class StateVerifierFlume extends StateVe .replaceAll(/^.*Master knows about [0-9]* nodes /,'') .trim(); shFlume.exec("connect localhost", - "exec config $node 'text(\"/etc/group\")' 'dfs(\"hdfs://localhost/flume.test\")'", + "exec config $node 'text(\"/etc/group\")' 'collectorSink(\"hdfs://localhost/flume\",\"test\")'", "quit\n"); sleep(5001); - (new Shell()).exec("hadoop fs -rm /flume.test"); } public boolean verifyState() { sleep(5001); - return ((new Shell()).exec("hadoop fs -ls /flume.test >/dev/null 2>&1").getRet() == 0); + boolean ret = ((new Shell()).exec("hadoop fs -ls /flume >/dev/null 2>&1").getRet() == 0); + (new Shell()).exec("hadoop fs -rmr /flume"); + return ret; } } \ No newline at end of file Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHBase.groovy URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHBase.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff ============================================================================== --- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHBase.groovy (original) +++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHBase.groovy Wed Nov 2 23:40:24 2011 @@ -36,6 +36,7 @@ class StateVerifierHBase extends StateVe public static void createStaticState() { shHBase.exec("create 't1', 'f1'", "put 't1', 'r1', 'f1:q', 'val'", + "flush 't1'", "quit\n"); } Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHDFS.groovy URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHDFS.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff ============================================================================== --- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHDFS.groovy (original) +++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHDFS.groovy Wed Nov 2 23:40:24 2011 @@ -55,6 +55,11 @@ class StateVerifierHDFS extends StateVer return (Arrays.equals(digest, MD5.digest())); } */ + public boolean config() { + Shell shRoot = new Shell("/bin/bash", "root"); + return 0 == shRoot.exec("sed -i -e 's#<configuration>\$#<configuration><property><name>dfs.safemode.min.datanodes</name><value>1</value></property><property><name>dfs.safemode.extension</name><value>0</value></property>#' /etc/hadoop/conf/hdfs-site.xml").getRet(); + } + public static void createStaticState() { sh.exec("hadoop fs -put <(echo StateHDFSVErifier) /StateHDFSVErifier"); } Copied: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHive.groovy (from r1196225, incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierFlume.groovy) URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHive.groovy?p2=incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHive.groovy&p1=incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierFlume.groovy&r1=1196225&r2=1196868&rev=1196868&view=diff ============================================================================== --- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierFlume.groovy (original) +++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHive.groovy Wed Nov 2 23:40:24 2011 @@ -15,34 +15,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.bigtop.itest.packagesmoke import org.apache.bigtop.itest.shell.Shell -import java.security.MessageDigest -import org.junit.Before -import org.junit.After -import org.junit.Test -class StateVerifierFlume extends StateVerifier { - Shell shFlume = new Shell("flume shell"); +class StateVerifierHive extends StateVerifier { + final static String schema = "CREATE TABLE state_table(A INT) PARTITIONED BY (dt STRING) row format delimited fields terminated by ',' escaped by '\\\\\\\\' stored as textfile;"; + + Shell sh = new Shell(); public void createState() { - String node; - shFlume.exec("connect localhost", - "getnodestatus", - "quit\n"); - node = shFlume.getOut().join(' ').replaceAll(/ --> IDLE.*$/,'') - .replaceAll(/^.*Master knows about [0-9]* nodes /,'') - .trim(); - shFlume.exec("connect localhost", - "exec config $node 'text(\"/etc/group\")' 'dfs(\"hdfs://localhost/flume.test\")'", - "quit\n"); - sleep(5001); - (new Shell()).exec("hadoop fs -rm /flume.test"); + File tmpFile = File.createTempFile("StateVerifierHiveData", ".txt"); + tmpFile.withWriter { (1..15).each { num -> it.write("$num\n"); } }; + + sh.exec("hive -e \"${schema}\""); + ["2008-08-08", "2008-08-09"].each { + sh.exec("hive -e \"LOAD DATA LOCAL INPATH '${tmpFile.getCanonicalPath()}' OVERWRITE INTO TABLE state_table partition (dt='$it');\""); + } + tmpFile.delete(); } public boolean verifyState() { - sleep(5001); - return ((new Shell()).exec("hadoop fs -ls /flume.test >/dev/null 2>&1").getRet() == 0); + sh.exec("hive -e 'SELECT COUNT(*) from state_table;'") + return (sh.getOut() =~ /30/).find(); } -} \ No newline at end of file +} Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHue.groovy URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHue.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff ============================================================================== --- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHue.groovy (original) +++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierHue.groovy Wed Nov 2 23:40:24 2011 @@ -36,20 +36,22 @@ class StateVerifierHue extends StateVeri public void createState() { // first call creates admin/admin username/keypair - sh.exec("curl --data '${creds}' ${loginURL}"); + sh.exec("curl -m 60 --data '${creds}' ${loginURL}"); } public boolean verifyState() { String sessionId; boolean res; - sh.exec("curl -i --data '${creds}' ${loginURL} | sed -e 's#Set-Cookie: *##' -e 's#;.*\$##' | grep '^sessionid'"); + sh.exec("curl -m 60 -i --data '${creds}' ${loginURL} | sed -e 's#Set-Cookie: *##' -e 's#;.*\$##' | grep '^sessionid'"); sessionId = sh.getOut().join(''); - res = (sh.exec("curl -b '${sessionId}' ${checkURL} | grep -q 'All ok. Configuration check passed'").getRet() == 0); + sh.exec("curl -m 60 -b '${sessionId}' ${checkURL}"); + res = (sh.getOut().grep( ~/.*All ok. Configuration check passed.*/ ).size() != 0) checkApps.each { - res = res && (sh.exec("curl -b '${sessionId}' ${hueServer}/${it}/ | grep -q 'Page Not Found'").getRet() != 0); + sh.exec("curl -m 60 -b '${sessionId}' ${hueServer}/${it}/"); + res = res && (sh.getOut().grep( ~/.*Page Not Found.*/ ).size() == 0); } return res; } -} \ No newline at end of file +} Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierMapreduce.groovy URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierMapreduce.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff ============================================================================== --- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierMapreduce.groovy (original) +++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/StateVerifierMapreduce.groovy Wed Nov 2 23:40:24 2011 @@ -28,7 +28,7 @@ class StateVerifierMapreduce extends Sta } public static boolean verifyStaticState() { - return sh.exec("hadoop jar `ls /usr/lib/hadoop-0.20/hadoop*examples*jar | head -1` pi 10 100").getRet() == 0; + return sh.exec("hadoop jar `ls /usr/lib/hadoop*/hadoop*examples*jar | head -1` pi 10 100").getRet() == 0; } void createState() { Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesBasics.groovy URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesBasics.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff ============================================================================== --- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesBasics.groovy (original) +++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesBasics.groovy Wed Nov 2 23:40:24 2011 @@ -125,10 +125,9 @@ class TestPackagesBasics extends Package @Test synchronized void testPackageInstall() { // WARNING: sometimes packages do not install because the server is busy - int i; - for (i=3; pkg.install() && i>0; i--) {}; - checkThat("could only install package $name on the ${3-i} try", - i, equalTo(3)); + for (int i=3; pkg.install() && i>0; i--) { + recordFailure("can not install pacakge $name will retry $i times"); + } // TODO: we need to come up with a way to abort any further execution to avoid spurious failures @@ -163,12 +162,6 @@ class TestPackagesBasics extends Package checkAlternatives(getMap(golden.alternatives)); } - @RunStage(level=1) - @Test - void testPackageRemove() { - checkRemoval(); - } - static void tryOrFail(Closure cl, int retries, String fail) { while (!cl.call()) { retries--; Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesPseudoDistributed.groovy URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesPseudoDistributed.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff ============================================================================== --- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesPseudoDistributed.groovy (original) +++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesPseudoDistributed.groovy Wed Nov 2 23:40:24 2011 @@ -23,7 +23,7 @@ import org.apache.bigtop.itest.junit.Ord import org.junit.Test @RunWith(OrderedParameterized.class) -class TestPackagesPseudoDistributed extends TestPackagesSingleNode { +class TestPackagesPseudoDistributed extends TestPackagesBasics { public TestPackagesPseudoDistributed(String pkgName, Node pkgGolden) { super(pkgName, pkgGolden); } @@ -33,8 +33,28 @@ class TestPackagesPseudoDistributed exte synchronized void testRemoteMetadata() { } - @RunStage(level=1) + @RunStage(level=-1) @Test - void testPackageRemove() { + void testPackageUpgrade() { + if (isUpgrade()) { + checkThat("upgrade sequence on a package $name failed to be executed", + CDHUpgradeSequence.execute(name, System.getProperty("cdh.prev.repo.version"), "3"), equalTo(0)); + } + } + + @Test + void testPulledDeps() { + checkPulledDeps(getMap(golden.deps)); + } + + @Test + void testPackageContent() { + Map files = getMap(golden.content); + checkFiles(files.config, files.doc, files.file); + } + + @Test + void testPackageServices() { + checkServices(getMap(golden.services)); } } Copied: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesPseudoDistributedWithRM.groovy (from r1196225, incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesReadiness.groovy) URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesPseudoDistributedWithRM.groovy?p2=incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesPseudoDistributedWithRM.groovy&p1=incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesReadiness.groovy&r1=1196225&r2=1196868&rev=1196868&view=diff ============================================================================== --- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesReadiness.groovy (original) +++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestPackagesPseudoDistributedWithRM.groovy Wed Nov 2 23:40:24 2011 @@ -15,58 +15,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.bigtop.itest.packagesmoke -import org.apache.bigtop.itest.junit.OrderedParameterized -import org.junit.runner.RunWith -import org.apache.bigtop.itest.junit.OrderedParameterized.RunStage import org.junit.Test -@RunWith(OrderedParameterized.class) -class TestPackagesReadiness extends TestPackagesSingleNode { - public TestPackagesReadiness(String pkgName, Node pkgGolden) { - super(pkgName, pkgGolden); - } - - @RunStage(level=-1) - @Test - void testPackageUpgrade() { - } - - @Test - void testRepoFile() { - } - - @Test - void testPulledDeps() { - } - - @Test - void testPackageMetadata() { - } - - @Test - void testPackageContent() { - } - - @Test - void testPackageServices() { - } +import static org.hamcrest.CoreMatchers.equalTo +import org.junit.runner.RunWith - @Test - void testUsers() { - } +import org.apache.bigtop.itest.junit.OrderedParameterized +import org.apache.bigtop.itest.junit.OrderedParameterized.RunStage - @Test - void testGroups() { - } +@RunWith(OrderedParameterized.class) +class TestPackagesPseudoDistributedWithRM extends TestPackagesPseudoDistributed { - @Test - void testAlternatives() { + public TestPackagesPseudoDistributedWithRM(String pkgName, Node pkgGolden) { + super(pkgName, pkgGolden); } @RunStage(level=1) @Test void testPackageRemove() { + checkComplimentary32bitInstall(); + checkDaemonStart(); + sleep(3001); // TODO FIXME: CDH-2816 should address the timing of daemons start. + checkRemoval(); } } Modified: incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestServices.groovy URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestServices.groovy?rev=1196868&r1=1196867&r2=1196868&view=diff ============================================================================== --- incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestServices.groovy (original) +++ incubator/bigtop/branches/hadoop-0.22/bigtop-tests/test-artifacts/package/src/main/groovy/org/apache/bigtop/itest/packagesmoke/TestServices.groovy Wed Nov 2 23:40:24 2011 @@ -28,12 +28,16 @@ import org.apache.bigtop.itest.posix.Ser import org.apache.bigtop.itest.junit.OrderedParameterized.RunStage import org.hamcrest.Matcher import static org.hamcrest.core.IsEqual.equalTo +import org.apache.bigtop.itest.shell.Shell @RunWith(OrderedParameterized.class) class TestServices { Map.Entry svcDescr; List<Service> svcs; StateVerifier verifier; + List<String> killIDs; + + static Shell shRoot = new Shell("/bin/bash", "root"); @Rule public ErrorCollector errors = new ErrorCollector(); @@ -49,6 +53,7 @@ class TestServices { svcDescr = svc; svcs = svcDescr.value.services.collect { new Service(it); }; verifier = svcDescr.value.verifier; + killIDs = svcDescr.value.killIDs; } static Map<String, Object[]> selectServices(String CDHrelease) { @@ -60,6 +65,14 @@ class TestServices { return res; } + @AfterClass + static void tearDown() { + // TODO: this is pretty silly, but it'll do for now + CDHServices.serviceDaemonUserNames.each { + shRoot.exec("kill -9 `ps -U${it} -opid=`"); + } + } + @RunStage(level=-1) @Test void createState() { @@ -79,16 +92,28 @@ class TestServices { verifier.verifyState(), equalTo(true)); svcs.reverseEach { - checkThat("service ${it.getName()} failed to stop", - it.stop(), equalTo(0)); + // TODO: we're only trying the best we can here + // there's a total eradication of services happening at @BeforeClass + it.stop(); + sleep(5001); } sleep(5001); + + // TODO: this is pretty silly, but it'll do for now + killIDs.each { + shRoot.exec("kill -9 `ps -U${it} -opid=`"); + } } @RunStage(level=1) @Test void verifyState() { svcs.each { + checkThat("failed to configure service ${it.getName()}", + verifier.config(), equalTo(true)); + } + + svcs.each { checkThat("service ${it.getName()} failed to start", it.start(), equalTo(0)); } @@ -97,6 +122,16 @@ class TestServices { verifier.verifyState(), equalTo(true)); svcs.reverseEach { it.stop(); } + sleep(5001); + // lets check if they are really stopped (if not -- we'll complain and kill them) + killIDs.each { + shRoot.exec("kill -0 `ps -U${it} -opid=`"); + if (!shRoot.getRet()) { + shRoot.exec("kill -9 `ps -U${it} -opid=`"); + checkThat("service running under the name of $it is supposed to be stopped, but it is not", + true, equalTo(false)); + } + } } public void checkThat(String msg, Object value, Matcher<Object> matcher) {
