This is an automated email from the ASF dual-hosted git repository. aonishuk pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 7c3275fde332585b5b3ce7c93afec7a84177498a Author: Andrew Onishuk <aonis...@hortonworks.com> AuthorDate: Thu Jan 10 08:20:26 2019 +0200 AMBARI-25095. deploy-gce-perf-cluster.py fails after upgrade on gce controller (aonishuk) --- agent.sh | 12 ++++++ contrib/utils/perf/deploy-gce-perf-cluster.py | 11 ++--- contrib/utils/perf/deploy-gce-perf-cluster.py.rej | 27 ++++++++++++ server.sh | 52 +++++++++++++++++++++++ 4 files changed, 97 insertions(+), 5 deletions(-) diff --git a/agent.sh b/agent.sh new file mode 100644 index 0000000..f1966f9 --- /dev/null +++ b/agent.sh @@ -0,0 +1,12 @@ +#!/bin/bash +yum install wget -y +wget -O /etc/yum.repos.d/ambari.repo http://10.240.0.30/ambari.repo +yum clean all; yum install krb5-workstation git ambari-agent -y +mkdir /home ; cd /home; git clone https://github.com/apache/ambari.git ; cd ambari ; git checkout branch-2.5 +cp -r /home/ambari/ambari-server/src/main/resources/stacks/PERF /var/lib/ambari-agent/cache/stacks/PERF +sed -i -f /var/lib/ambari-agent/cache/stacks/PERF/PythonExecutor.sed /usr/lib/ambari-agent/lib/ambari_agent/PythonExecutor.py +sed -i -f /var/lib/ambari-agent/cache/stacks/PERF/check_host.sed /var/lib/ambari-agent/cache/custom_actions/scripts/check_host.py +sed -i -e 's/hostname=localhost/hostname=perf-server-test-perf-1.c.pramod-thangali.internal/g' /etc/ambari-agent/conf/ambari-agent.ini +sed -i -e 's/agent]/agent]\nhostname_script=foo\npublic_hostname_script=foo\n/1' /etc/ambari-agent/conf/ambari-agent.ini +wget http://10.240.0.30/agent-multiplier.py ; python /home/ambari/ambari-agent/conf/unix/agent-multiplier.py start +exit 0 \ No newline at end of file diff --git a/contrib/utils/perf/deploy-gce-perf-cluster.py b/contrib/utils/perf/deploy-gce-perf-cluster.py index a1259eb..e34b07c 100644 --- a/contrib/utils/perf/deploy-gce-perf-cluster.py +++ b/contrib/utils/perf/deploy-gce-perf-cluster.py @@ -475,22 +475,23 @@ def get_vms_list(args): def __get_vms_list_from_name(args, cluster_name): """ - Method to parse "gce fqdn {cluster-name}" command output and get hosts and ips pairs for every host in cluster + Method to parse "gce info {cluster-name}" command output and get hosts and ips pairs for every host in cluster :param args: Command line args :return: Mapping of VM host name to ip. """ - gce_fqdb_cmd = '/opt/gce-utils/gce fqdn {0}'.format(cluster_name) + gce_fqdb_cmd = '/opt/gce-utils/gce info {0}'.format(cluster_name) out = execute_command(args, args.controller, gce_fqdb_cmd, "Failed to get VMs list!", "-tt") lines = out.split('\n') #print "LINES=" + str(lines) if lines[0].startswith("Using profile") and not lines[1].strip(): result = {} - for s in lines[2:]: # Ignore non-meaningful lines + for s in lines[4:]: # Ignore non-meaningful lines if not s: continue - match = re.match(r'^([\d\.]*)\s+([\w\.-]*)\s+([\w\.-]*)\s+$', s, re.M) + + match = re.match(r'^ [^ ]+ ([\w\.-]*)\s+([\d\.]*).*$', s, re.M) if match: - result[match.group(2)] = match.group(1) + result[match.group(1)] = match.group(2) else: raise Exception('Cannot parse "{0}"'.format(s)) return result diff --git a/contrib/utils/perf/deploy-gce-perf-cluster.py.rej b/contrib/utils/perf/deploy-gce-perf-cluster.py.rej new file mode 100644 index 0000000..06eca6e --- /dev/null +++ b/contrib/utils/perf/deploy-gce-perf-cluster.py.rej @@ -0,0 +1,27 @@ +--- contrib/utils/perf/deploy-gce-perf-cluster.py ++++ contrib/utils/perf/deploy-gce-perf-cluster.py +@@ -28,7 +28,7 @@ import re + import socket + + cluster_prefix = "perf" +-ambari_repo_file_url = "http://s3.amazonaws.com/dev.hortonworks.com/ambari/centos6/2.x/updates/2.5.0.0/ambaribn.repo" ++ambari_repo_file_url = "http://10.240.0.30/ambari.repo" + + public_hostname_script = "foo" + hostname_script = "foo" +@@ -397,13 +394,11 @@ def create_agent_script(server_host_name): + # TODO, instead of cloning Ambari repo on each VM, do it on the server once and distribute to all of the agents. + contents = "#!/bin/bash\n" + \ + "wget -O /etc/yum.repos.d/ambari.repo {0}\n".format(ambari_repo_file_url) + \ +- "yum clean all; yum install krb5-workstation git ambari-agent -y\n" + \ +- "mkdir /home ; cd /home; git clone https://github.com/apache/ambari.git ; cd ambari ; git checkout branch-2.5\n" + \ +- "cp -r /home/ambari/ambari-server/src/main/resources/stacks/PERF /var/lib/ambari-agent/cache/stacks/PERF\n" + \ ++ "yum clean all; yum install krb5-workstation ambari-agent -y\n" + \ + "sed -i -f /var/lib/ambari-agent/cache/stacks/PERF/PythonExecutor.sed /usr/lib/python2.6/site-packages/ambari_agent/PythonExecutor.py\n" + \ + "sed -i -e 's/hostname=localhost/hostname={0}/g' /etc/ambari-agent/conf/ambari-agent.ini\n".format(server_host_name) + \ + "sed -i -e 's/agent]/agent]\\nhostname_script={0}\\npublic_hostname_script={1}\\n/1' /etc/ambari-agent/conf/ambari-agent.ini\n".format(hostname_script, public_hostname_script) + \ +- "python /home/ambari/ambari-agent/conf/unix/agent-multiplier.py start\n" + \ ++ "wget http://10.240.0.30/agent-multiplier.py ; python agent-multiplier.py start\n" + \ + "exit 0" + + with open("agent.sh", "w") as f: diff --git a/server.sh b/server.sh new file mode 100644 index 0000000..880ab44 --- /dev/null +++ b/server.sh @@ -0,0 +1,52 @@ +#!/bin/bash +yum install wget -y +wget -O /etc/yum.repos.d/ambari.repo http://10.240.0.30/ambari.repo +yum clean all; yum install ambari-server -y +sed -i -f /home/ambari/ambari-server/src/main/resources/stacks/PERF/install_packages.sed /var/lib/ambari-server/resources/custom_actions/scripts/install_packages.py +sed -i -f /home/ambari/ambari-server/src/main/resources/stacks/PERF/install_packages.sed /var/lib/ambari-agent/cache/custom_actions/scripts/install_packages.py + + +cd /; wget http://central.maven.org/maven2/mysql/mysql-connector-java/5.1.40/mysql-connector-java-5.1.40.jar; +mkdir /usr/share/java; chmod 777 /usr/share/java;cp mysql-connector-java-5.1.40.jar /usr/share/java/; chmod 777 /usr/share/java/mysql-connector-java-5.1.40.jar; +ln -s /usr/share/java/mysql-connector-java-5.1.40.jar /usr/share/java/mysql-connector-java.jar; +cd /etc/yum.repos.d/; wget http://repo.mysql.com/mysql-community-release-el6-5.noarch.rpm; rpm -ivh mysql-community-release-el6-5.noarch.rpm;yum clean all; yum install mysql-server -y +sed -i -e 's/mysqld]/mysqld]\nmax_allowed_packet=1024M\njoin_buffer_size=512M\nsort_buffer_size=128M\nread_rnd_buffer_size=128M\ninnodb_buffer_pool_size=16G\ninnodb_file_io_threads=16\ninnodb_thread_concurrency=32\nkey_buffer_size=16G\nquery_cache_limit=16M\nquery_cache_size=512M\nthread_cache_size=128\ninnodb_log_buffer_size=512M/1' /etc/my.cnf +service mysqld start +mysql -uroot -e "CREATE DATABASE ambari;" +mysql -uroot -e "SOURCE /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql;" ambari +mysql -uroot -e "CREATE USER 'ambari'@'%' IDENTIFIED BY 'bigdata';" +mysql -uroot -e "GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'%%';" +mysql -uroot -e "CREATE USER 'ambari'@'localhost' IDENTIFIED BY 'bigdata';" +mysql -uroot -e "GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'localhost';" +mysql -uroot -e "CREATE USER 'ambari'@'perf-server-test-perf-1.c.pramod-thangali.internal' IDENTIFIED BY 'bigdata';" +mysql -uroot -e "GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'perf-server-test-perf-1.c.pramod-thangali.internal';" +mysql -uroot -e "FLUSH PRIVILEGES;" + + +ambari-server setup -s +ambari-server setup --database mysql --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar --databasehost=localhost --databaseport=3306 --databasename=ambari --databaseusername=ambari --databasepassword=bigdata +sed -i -e 's/=postgres/=mysql/g' /etc/ambari-server/conf/ambari.properties +sed -i -e 's/server.persistence.type=local/server.persistence.type=remote/g' /etc/ambari-server/conf/ambari.properties +sed -i -e 's/local.database.user=postgres//g' /etc/ambari-server/conf/ambari.properties +sed -i -e 's/server.jdbc.postgres.schema=ambari//g' /etc/ambari-server/conf/ambari.properties +sed -i -e 's/agent.threadpool.size.max=25/agent.threadpool.size.max=100/g' /etc/ambari-server/conf/ambari.properties +sed -i -e 's/client.threadpool.size.max=25/client.threadpool.size.max=65/g' /etc/ambari-server/conf/ambari.properties +sed -i -e 's/false/true/g' /var/lib/ambari-server/resources/stacks/PERF/1.0/metainfo.xml +sed -i -e 's/false/true/g' /var/lib/ambari-server/resources/stacks/PERF/2.0/metainfo.xml +sed -i -e 's/-Xmx2048m/-Xmx16384m/g' /var/lib/ambari-server/ambari-env.sh + +echo 'server.jdbc.driver=com.mysql.jdbc.Driver' >> /etc/ambari-server/conf/ambari.properties +echo 'server.jdbc.rca.url=jdbc:mysql://perf-server-test-perf-1.c.pramod-thangali.internal:3306/ambari' >> /etc/ambari-server/conf/ambari.properties +echo 'server.jdbc.rca.driver=com.mysql.jdbc.Driver' >> /etc/ambari-server/conf/ambari.properties +echo 'server.jdbc.url=jdbc:mysql://perf-server-test-perf-1.c.pramod-thangali.internal:3306/ambari' >> /etc/ambari-server/conf/ambari.properties +echo 'server.jdbc.port=3306' >> /etc/ambari-server/conf/ambari.properties +echo 'server.jdbc.hostname=localhost' >> /etc/ambari-server/conf/ambari.properties +echo 'server.jdbc.driver.path=/usr/share/java/mysql-connector-java.jar' >> /etc/ambari-server/conf/ambari.properties +echo 'alerts.cache.enabled=true' >> /etc/ambari-server/conf/ambari.properties +echo 'alerts.cache.size=100000' >> /etc/ambari-server/conf/ambari.properties +echo 'alerts.execution.scheduler.maxThreads=4' >> /etc/ambari-server/conf/ambari.properties +echo 'security.temporary.keystore.retention.minutes=180' >> /etc/ambari-server/conf/ambari.properties +echo 'stack.hooks.folder=stacks/PERF/1.0/hooks' >> /etc/ambari-server/conf/ambari.properties + +ambari-server start --skip-database-check +exit 0 \ No newline at end of file