Cory Johns has proposed merging lp:~bigdata-dev/charms/trusty/apache-hive/trunk into lp:charms/trusty/apache-hive.
Requested reviews: charmers (charmers) For more details, see: https://code.launchpad.net/~bigdata-dev/charms/trusty/apache-hive/trunk/+merge/273615 Test cleanups for CWR, and moving binaries to S3 -- Your team Juju Big Data Development is subscribed to branch lp:~bigdata-dev/charms/trusty/apache-hive/trunk.
=== modified file 'README.md' --- README.md 2015-08-24 23:23:36 +0000 +++ README.md 2015-10-06 20:46:56 +0000 @@ -84,7 +84,7 @@ ## Contact Information -- <[email protected]> +- <[email protected]> ## Help === modified file 'resources.yaml' --- resources.yaml 2015-08-24 23:23:36 +0000 +++ resources.yaml 2015-10-06 20:46:56 +0000 @@ -7,10 +7,10 @@ pypi: jujubigdata>=4.0.0,<5.0.0 optional_resources: hive-ppc64le: - url: https://git.launchpad.net/bigdata-data/plain/apache/ppc64le/apache-hive-0.13.0-bin.tar.gz?id=c34a21c939f5fce9ab89b95d65fe2df50e7bbab0 + url: https://s3.amazonaws.com/jujubigdata/apache/ppc64le/apache-hive-0.13.0-bin-4c83564.tar.gz hash: 4c835644eb72a08df059b86c45fb159b95df08e831334cb57e24654ef078e7ee hash_type: sha256 hive-x86_64: - url: https://git.launchpad.net/bigdata-data/plain/apache/x86_64/apache-hive-1.0.0-bin.tar.gz?id=c34a21c939f5fce9ab89b95d65fe2df50e7bbab0 + url: https://s3.amazonaws.com/jujubigdata/apache/x86_64/apache-hive-1.0.0-bin-b8e121f.tar.gz hash: b8e121f435defeb94d810eb6867d2d1c27973e4a3b4099f2716dbffafb274184 hash_type: sha256 === removed file 'tests/00-setup' --- tests/00-setup 2015-09-16 21:43:12 +0000 +++ tests/00-setup 1970-01-01 00:00:00 +0000 @@ -1,8 +0,0 @@ -#!/bin/bash - -if ! dpkg -s amulet &> /dev/null; then - echo Installing Amulet... - sudo add-apt-repository -y ppa:juju/stable - sudo apt-get update - sudo apt-get -y install amulet -fi === added file 'tests/01-basic-deployment.py' --- tests/01-basic-deployment.py 1970-01-01 00:00:00 +0000 +++ tests/01-basic-deployment.py 2015-10-06 20:46:56 +0000 @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +import unittest +import amulet + + +class TestDeploy(unittest.TestCase): + """ + Trivial deployment test for Apache Hive. + + This charm cannot do anything useful by itself, so integration testing + is done in the bundle. + """ + + def test_deploy(self): + self.d = amulet.Deployment(series='trusty') + self.d.add('hive', 'apache-hive') + self.d.setup(timeout=900) + self.d.sentry.wait(timeout=1800) + self.unit = self.d.sentry['hive'][0] + + +if __name__ == '__main__': + unittest.main() === removed file 'tests/100-deploy-hive-mysql' --- tests/100-deploy-hive-mysql 2015-09-17 14:17:43 +0000 +++ tests/100-deploy-hive-mysql 1970-01-01 00:00:00 +0000 @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 -import unittest -import amulet - - -class TestDeploy(unittest.TestCase): - """ - Deployment test for Apache Hive using HDFS as shared storage and YARN as - cluster job manager. - """ - - @classmethod - def setUpClass(cls): - cls.d = amulet.Deployment(series='trusty') - # Deploy a hadoop cluster - cls.d.add('yarn-master', charm='cs:trusty/apache-hadoop-yarn-master') - cls.d.add('hdfs-master', charm='cs:trusty/apache-hadoop-hdfs-master') - cls.d.add('compute-slave', charm='cs:trusty/apache-hadoop-compute-slave') - cls.d.add('plugin', charm='cs:trusty/apache-hadoop-plugin') - cls.d.relate('yarn-master:namenode', 'hdfs-master:namenode') - cls.d.relate('compute-slave:nodemanager', 'yarn-master:nodemanager') - cls.d.relate('compute-slave:datanode', 'hdfs-master:datanode') - cls.d.relate('plugin:resourcemanager', 'yarn-master:resourcemanager') - cls.d.relate('plugin:namenode', 'hdfs-master:namenode') - - # Add MySQL service (hive needs binlog-format config) - cls.d.add('mysql', 'cs:trusty/mysql') - cls.d.configure('mysql', {'binlog-format': 'row'}) - - # Add Hive service - cls.d.add('hive', charm='cs:trusty/apache-hive') - cls.d.relate('hive:db', 'mysql:db') - cls.d.relate('hive:hadoop-plugin', 'plugin:hadoop-plugin') - - cls.d.setup(timeout=3600) - cls.d.sentry.wait(timeout=3600) - cls.unit = cls.d.sentry.unit['hive/0'] - - -########################################################################### -# Validate yarn mapreduce operation from the Hive node -# 1) validate mapreduce execution - writing to hdfs -########################################################################### - def test_yarn_mapreduce_exe1(self): - outdir = "/user/ubuntu/teragenout" - o, c = self.unit.run("su ubuntu -c 'hdfs dfs -rm -f -R %s &&" - "hadoop jar /usr/lib/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples*.jar teragen 10000 %s'" - % (outdir, outdir)) - assert c == 0, "Teragen failed to execute: %s" % o - -########################################################################### -# 2) validate successful mapreduce operation after the execution -########################################################################### - o, c = self.unit.run("su hdfs -c 'hdfs dfs -ls %s/_SUCCESS'" % outdir) - assert c == 0, "Teragen executed, but expected output was not found: %s" % o - -########################################################################### -# Validate mapreduce operation from Hive node - validates job chain operation -# 1) validate mapreduce execution - reading and writing to hdfs -########################################################################### - def test_yarn_mapreduce_exe2(self): - indir = "/user/ubuntu/teragenout" - outdir = "/user/ubuntu/terasortout" - o, c = self.unit.run("su ubuntu -c 'hdfs dfs -rm -f -R %s &&" - "hadoop jar /usr/lib/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples*.jar terasort %s %s'" - % (outdir, indir, outdir)) - assert c == 0, "Terasort failed to execute: %s" % o - -########################################################################### -# 2) validate a successful mapreduce operation after the execution -########################################################################### - o, c = self.unit.run("su hdfs -c 'hdfs dfs -ls %s/_SUCCESS'" % outdir) - assert c == 0, "Terasort executed, but expected output was not found: %s" % o - -########################################################################### -# Validate the service is running -# Validate the JVM -########################################################################### - def test_jvm_status(self): - o, c = self.unit.run("su hive -c 'pgrep -a java | grep HiveServer2'") - assert "HiveServer2" in o, "HiveServer2 not running" - -########################################################################### -# Validate HIVE command line operation - create a HIVE table -########################################################################### - def test_hive_create_tables(self): - o, c = self.unit.run("su hive -c \"hive -e 'create table test(col1 int, col2 string);' 2>&1\"") - assert c == 0, "Hive create table failed: %s" % o - -########################################################################### -# Validate HIVE commandline operation - show HIVE tables -########################################################################### - def test_hive_show_tables(self): - o, c = self.unit.run("su hive -c \"hive -e \'show tables;\' 2>&1\"") - # Look for the 'test' table from our previous test - assert "test" in o, "Hive show tables failed: %s" % o - - -if __name__ == '__main__': - unittest.main() === removed directory 'tests/remote' === removed file 'tests/remote/test_dist_config.py' --- tests/remote/test_dist_config.py 2015-08-21 21:51:45 +0000 +++ tests/remote/test_dist_config.py 1970-01-01 00:00:00 +0000 @@ -1,72 +0,0 @@ -#!/usr/bin/env python - -import grp -import os -import pwd -import unittest - -import jujubigdata - - -class TestDistConfig(unittest.TestCase): - """ - Test that the ``dist.yaml`` settings were applied properly, such as users, groups, and dirs. - - This is done as a remote test on the deployed unit rather than a regular - test under ``tests/`` because filling in the ``dist.yaml`` requires Juju - context (e.g., config). - """ - @classmethod - def setUpClass(cls): - config = None - config_dir = os.environ['JUJU_CHARM_DIR'] - config_file = 'dist.yaml' - if os.path.isfile(os.path.join(config_dir, config_file)): - config = os.path.join(config_dir, config_file) - if not config: - raise IOError('Could not find {} in {}'.format(config_file, config_dir)) - reqs = ['vendor', 'hadoop_version', 'packages', 'groups', 'users', - 'dirs', 'ports'] - cls.dist_config = jujubigdata.utils.DistConfig(config, reqs) - - def test_groups(self): - for name in self.dist_config.groups: - try: - grp.getgrnam(name) - except KeyError: - self.fail('Group {} is missing'.format(name)) - - def test_users(self): - for username, details in self.dist_config.users.items(): - try: - user = pwd.getpwnam(username) - except KeyError: - self.fail('User {} is missing'.format(username)) - for groupname in details['groups']: - try: - group = grp.getgrnam(groupname) - except KeyError: - self.fail('Group {} referenced by user {} does not exist'.format( - groupname, username)) - if group.gr_gid != user.pw_gid: - self.assertIn(username, group.gr_mem, 'User {} not in group {}'.format( - username, groupname)) - - def test_dirs(self): - for name, details in self.dist_config.dirs.items(): - dirpath = self.dist_config.path(name) - self.assertTrue(dirpath.isdir(), 'Dir {} is missing'.format(name)) - stat = dirpath.stat() - owner = pwd.getpwuid(stat.st_uid).pw_name - group = grp.getgrgid(stat.st_gid).gr_name - perms = stat.st_mode & ~0o40000 - self.assertEqual(owner, details.get('owner', 'root'), - 'Dir {} ({}) has wrong owner: {}'.format(name, dirpath, owner)) - self.assertEqual(group, details.get('group', 'root'), - 'Dir {} ({}) has wrong group: {}'.format(name, dirpath, group)) - self.assertEqual(perms, details.get('perms', 0o755), - 'Dir {} ({}) has wrong perms: 0o{:o}'.format(name, dirpath, perms)) - - -if __name__ == '__main__': - unittest.main() === modified file 'tests/tests.yaml' --- tests/tests.yaml 2015-06-25 15:00:26 +0000 +++ tests/tests.yaml 2015-10-06 20:46:56 +0000 @@ -1,10 +1,3 @@ -# Driver for bundletester: https://github.com/juju-solutions/bundletester -# -# It may be useful to alter the defaults during manual testing. For example, -# set 'reset: false' to reuse existing charms instead of redeploying them. - -# Allow bootstrap of current env, default: true -bootstrap: true - -# Use juju-deployer to reset env between test, default: true -reset: true +reset: false +packages: + - amulet
-- Mailing list: https://launchpad.net/~bigdata-dev Post to : [email protected] Unsubscribe : https://launchpad.net/~bigdata-dev More help : https://help.launchpad.net/ListHelp
