This is an automated email from the ASF dual-hosted git repository. jialiang pushed a commit to branch docker_ambari in repository https://gitbox.apache.org/repos/asf/bigtop.git
commit 9292a98bf69634c5839197ae2cdb473a04e71585 Author: jialiang <[email protected]> AuthorDate: Thu Aug 8 14:11:11 2024 +0800 fix --- provisioner/docker/docker-compose.yml | 2 + provisioner/docker/docker-hadoop.sh | 84 ++++++++++++++++++----------------- provisioner/utils/install_cluster.sh | 39 ++++++++++++++++ provisioner/utils/setup-env-centos.sh | 70 +++++++++++++++++++++++------ 4 files changed, 141 insertions(+), 54 deletions(-) diff --git a/provisioner/docker/docker-compose.yml b/provisioner/docker/docker-compose.yml index ffab94e5e..f087365dd 100644 --- a/provisioner/docker/docker-compose.yml +++ b/provisioner/docker/docker-compose.yml @@ -26,3 +26,5 @@ services: - ./config/hieradata:/etc/puppet/hieradata - ./config/hosts:/etc/hosts - /sys/fs/cgroup:/sys/fs/cgroup:ro + ports: + - "${HOST_PORT}-${HOST_PORT_END}:8080" diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh index f235ac9ae..8912b3612 100755 --- a/provisioner/docker/docker-hadoop.sh +++ b/provisioner/docker/docker-hadoop.sh @@ -28,7 +28,6 @@ usage() { echo " -e, --exec INSTANCE_NO|INSTANCE_NAME - Execute command on a specific instance. Instance can be specified by name or number" echo " For example: $PROG --exec 1 bash" echo " $PROG --exec docker_bigtop_1 bash" - echo " -E, --env-check - Check whether required tools has been installed" echo " -G, --disable-gpg-check - Disable gpg check for the Bigtop repository" echo " -k, --stack COMPONENTS - Overwrite the components to deploy defined in config file" echo " COMPONENTS is a comma separated string" @@ -37,16 +36,7 @@ usage() { echo " -l, --list - List out container status for the cluster" echo " -L, --enable-local-repo - Whether to use repo created at local file system. You can get one by $ ./gradlew repo" echo " -m, --memory MEMORY_LIMIT - Overwrite the memory_limit defined in config file" - echo " -n, --nexus NEXUS_URL - Configure Nexus proxy to speed up test execution" - echo " NEXUS_URL is optional. If not specified, default to http://NEXUS_IP:8081/nexus"; - echo " Where NEXUS_IP is the ip of container named nexus" - echo " -p, --provision - Deploy configuration changes" echo " -r, --repo REPO_URL - Overwrite the yum/apt repo defined in config file" - echo " -s, --smoke-tests COMPONENTS - Run Bigtop smoke tests" - echo " COMPONENTS is optional. If not specified, default to smoke_test_components in config file" - echo " COMPONENTS is a comma separated string" - echo " For example: $PROG -c 3 --smoke-tests hdfs" - echo " $PROG -c 3 --smoke-tests 'hdfs, yarn, mapreduce'" echo " -h, --help" exit 1 } @@ -79,6 +69,8 @@ create() { fi export MEM_LIMIT=${memory_limit} + export HOST_PORT=8082 + export HOST_PORT HOST_PORT_END=8089 # Startup instances $DOCKER_COMPOSE_CMD -p $PROVISION_ID up -d --scale bigtop=$1 --no-recreate if [ $? -ne 0 ]; then @@ -120,12 +112,51 @@ create() { gpg_check=true fi fi - generate-config "$hadoop_head_node" "$repo" "$components" +# generate-config "$hadoop_head_node" "$repo" "$components" # Start provisioning generate-hosts bootstrap $distro $enable_local_repo - provision + + DEST_DIR="/bigtop-home/bigdata-deploy" + CONF_DIR="/bigtop-home/bigdata-deploy/conf" + FILENAME="base_conf.yml" + + output="" + for node in ${NODES[*]}; do + ip_hostname=$(docker inspect --format "{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}} {{.Config.Hostname}}" $node) + output+=" - $ip_hostname B767610qa4Z\n" + done + + + + echo "Configuration file has been generated at $CONF_DIR/$FILENAME" + + get_nodes + for node in ${NODES[*]}; do + docker exec $node bash -c "echo -e \"default_password: 'B767610qa4Z' +stack_version: '3.3.0' +data_dirs: ['/data/sdv1'] +repos: + +user: root +hosts: +$output +components_to_install: ['zookeeper','ambari'] + +backup_old_repo: no +should_deploy_ambari_mpack: false +deploy_ambari_only: false +prepare_nodes_only: false + +cluster_name: 'cluster' +hdfs_ha_name: 'c1' +ansible_ssh_port: 22\" > $CONF_DIR/$FILENAME" & + docker exec $node bash -c "/bigtop-home/provisioner/utils/install_cluster.sh $DEST_DIR" & + break # 只在第一台机器上执行 + done + wait +# provision } generate-hosts() { @@ -367,12 +398,6 @@ while [ $# -gt 0 ]; do shift execute "$@" shift $#;; - -E|--env-check) - env-check - shift;; - -G|--disable-gpg-check) - gpg_check=false - shift;; -k|--stack) if [ $# -lt 2 ]; then log "No stack specified" @@ -413,20 +438,6 @@ while [ $# -gt 0 ]; do fi memory_limit=$2 shift 2;; - -n|--nexus) - if [ $# -lt 2 ] || [[ $2 == -* ]]; then - NEXUS_IP=`docker inspect --format "{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}" nexus` - if [ $? != 0 ]; then - log "No container named nexus exists. To create one:\n $ docker run -d --name nexus sonatype/nexus" - exit 1 - fi - configure-nexus "http://$NEXUS_IP:8081/nexus"; - shift - else - configure-nexus $2 - shift 2 - fi - ;; -p|--provision) provision shift;; @@ -437,15 +448,6 @@ while [ $# -gt 0 ]; do fi repo=$2 shift 2;; - -s|--smoke-tests) - if [ $# -lt 2 ] || [[ $2 == -* ]]; then - shift - else - smoke_test_components=$2 - shift 2 - fi - READY_TO_TEST=true - ;; -h|--help) usage shift;; diff --git a/provisioner/utils/install_cluster.sh b/provisioner/utils/install_cluster.sh new file mode 100644 index 000000000..9773c5bda --- /dev/null +++ b/provisioner/utils/install_cluster.sh @@ -0,0 +1,39 @@ +#!/bin/bash + + +if [ $# -eq 0 ]; then + echo "Error: Please provide a directory as an argument." + echo "Usage: $0 <directory>" + exit 1 +fi + +DEST_DIR="$1" + +if [ ! -d "$DEST_DIR" ]; then + echo "Error: Directory '$DEST_DIR' does not exist." + exit 1 +fi + +SCRIPT_DIR=$DEST_DIR +cd "$SCRIPT_DIR" + +# 激活虚拟环境 +if [ -f "venv.sh" ]; then + source venv.sh +else + echo "Warning: venv.sh not found in the current directory." +fi + +python3 ci_tools/main.py -generate-conf +if [ $? -ne 0 ]; then + echo "Error: Failed to generate configuration." + exit 1 +fi + +python3 ci_tools/main.py -deploy +if [ $? -ne 0 ]; then + echo "Error: Deployment failed." + exit 1 +fi + +echo "Deployment completed successfully." diff --git a/provisioner/utils/setup-env-centos.sh b/provisioner/utils/setup-env-centos.sh index 1b85eef76..d0e4a0760 100755 --- a/provisioner/utils/setup-env-centos.sh +++ b/provisioner/utils/setup-env-centos.sh @@ -17,6 +17,9 @@ enable_local_repo=${1:-false} +mkdir -p /data/sdv1 +echo 'root:B767610qa4Z' | chpasswd + # This may be crazy, but unless we change this - RHEL will actively # revert back to localhost.localdomain sed -ie 's#HOSTNAME=.*$#HOSTNAME='`hostname -f`'#' /etc/sysconfig/network @@ -25,25 +28,65 @@ if [ -f /etc/os-release ]; then . /etc/os-release fi + +cd /etc/yum.repos.d/ && mkdir backup && mv *repo backup/ +repo_content='[baseos] +name=Rocky Linux $releasever - BaseOS +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=$basearch&repo=BaseOS-$releasever +gpgcheck=1 +enabled=1 +countme=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial + +[devel] +name=Rocky Linux $releasever - Devel WARNING! FOR BUILDROOT AND KOJI USE +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=$basearch&repo=Devel-$releasever +gpgcheck=1 +enabled=1 +countme=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial +' + +# Write the content to /etc/yum.repos.d/RockylinuxBase.repo +echo "$repo_content" > /etc/yum.repos.d/RockylinuxBase.repo + +# Print a message indicating completion +echo "Repository configuration has been written to /etc/yum.repos.d/RockylinuxBase.repo" + +#curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-8.repo + +#sed -e 's|^mirrorlist=|#mirrorlist=|g' \ +# -e 's|^#baseurl=http://dl.rockylinux.org/$contentdir|baseurl=https://mirrors.aliyun.com/rockylinux|g' \ +# -i.bak \ +# /etc/yum.repos.d/Rocky*.repo + +dnf makecache +yum install -y sshpass +yum -y install python3-devel +yum -y install createrepo +yum -y install openssh-clients openssh-server +systemctl start sshd.service + +systemctl unmask systemd-logind.service # Setup rng-tools to improve virtual machine entropy performance. # The poor entropy performance will cause kerberos provisioning failed. # BIGTOP-3883: # yum-utils, yum-priorities and yum-config-manager are NOT available in openEuler 22.03 -if [ "${ID}" = "openEuler" ];then - dnf install rng-tools -y -else - yum -y install rng-tools yum-priorities -fi +#if [ "${ID}" = "openEuler" ];then +# dnf install rng-tools -y +#else +# yum -y install yum-priorities +#fi -if [ -x /usr/bin/systemctl ] ; then - sed -i 's@ExecStart=/sbin/rngd -f@ExecStart=/sbin/rngd -f -r /dev/urandom@' /usr/lib/systemd/system/rngd.service - systemctl daemon-reload - systemctl start rngd -else - sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd - service rngd start -fi +#if [ -x /usr/bin/systemctl ] ; then +# sed -i 's@ExecStart=/sbin/rngd -f@ExecStart=/sbin/rngd -f -r /dev/urandom@' /usr/lib/systemd/system/rngd.service +# systemctl daemon-reload +# systemctl start rngd +#else +# sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd +# service rngd start +#fi if [ $enable_local_repo == "true" ]; then echo "Enabling local yum." @@ -71,3 +114,4 @@ if [ $enable_local_repo == "true" ]; then else echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..." fi +
