This is an automated email from the ASF dual-hosted git repository. ywkim pushed a commit to branch cnb in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/cnb by this push: new dccbc42 BIGTOP-3238: Local muti-node cluster via vagrant and kubespray dccbc42 is described below commit dccbc42270058fe29af5223bf8e6b642320be469 Author: Youngwoo Kim <yw...@apache.org> AuthorDate: Mon Oct 14 10:29:58 2019 +0900 BIGTOP-3238: Local muti-node cluster via vagrant and kubespray --- README.md | 38 +++++++ bigtop.bom | 12 +++ build.gradle | 2 +- kubespray/vagrant/Vagrantfile | 227 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 278 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5bec287..0db5be6 100755 --- a/README.md +++ b/README.md @@ -1,3 +1,41 @@ +[![Travis CI](https://img.shields.io/travis/apache/bigtop.svg?branch=master)](https://travis-ci.org/apache/bigtop) + +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +[Apache Bigtop](http://bigtop.apache.org/) +========================================== + +TBD + +# Get Started with Deployment and Smoke Testing of Cloud Native BigTop + +Prerequisites: +- Vagrant +- Java + +## Set up 3-Node Kubernetes cluster via Kubespray on local machine +``` +$ cd $BIGTOP_HOME +$ ./gradlew kubespray-clean kubespray-download && cd dl/ && tar xvfz kubespray-2.11.0.tar.gz +$ cd dl/kubespray-2.11.0/ && cp ../../kubespray/vagrant/Vagrantfile . +$ vagrant up +``` + +# Cloud Native Bigtop This is the content for the talk given by jay vyas and sid mani @ apachecon 2019 in Las Vegas, you can watch it here https://www.youtube.com/watch?v=LUCE63q ! # TLDR, heres how you create an analytics distro on K8s... diff --git a/bigtop.bom b/bigtop.bom index bafca95..99ab5c8 100644 --- a/bigtop.bom +++ b/bigtop.bom @@ -122,6 +122,18 @@ bigtop { ] components { + 'kubespray' { + name = "kubespray" + pkg = "kubespray" + relNotes = "" + website = "https://github.com/kubernetes-sigs/kubespray" + version { base = '2.11.0'; pkg = base; release = 1 } + tarball { destination = "$name-${version.base}.tar.gz" + source = "v${version.base}.tar.gz" } + url { site = "https://github.com/kubernetes-sigs/kubespray/archive/" + archive = site } + } + 'zookeeper' { name = 'zookeeper' pkg = name diff --git a/build.gradle b/build.gradle index 8dc9095..a9065fd 100644 --- a/build.gradle +++ b/build.gradle @@ -23,7 +23,7 @@ buildscript { } plugins { - id "de.undercouch.download" version "3.2.0" + id "de.undercouch.download" version "4.0.0" id "org.nosphere.apache.rat" version "0.2.0" } diff --git a/kubespray/vagrant/Vagrantfile b/kubespray/vagrant/Vagrantfile new file mode 100644 index 0000000..9702059 --- /dev/null +++ b/kubespray/vagrant/Vagrantfile @@ -0,0 +1,227 @@ +# -*- mode: ruby -*- +# # vi: set ft=ruby : + +# For help on using kubespray with vagrant, check out docs/vagrant.md + +require 'fileutils' + +Vagrant.require_version ">= 2.0.0" + +CONFIG = File.join(File.dirname(__FILE__), "vagrant/config.rb") + +COREOS_URL_TEMPLATE = "https://storage.googleapis.com/%s.release.core-os.net/amd64-usr/current/coreos_production_vagrant.json" + +# Uniq disk UUID for libvirt +DISK_UUID = Time.now.utc.to_i + +SUPPORTED_OS = { + "coreos-stable" => {box: "coreos-stable", user: "core", box_url: COREOS_URL_TEMPLATE % ["stable"]}, + "coreos-alpha" => {box: "coreos-alpha", user: "core", box_url: COREOS_URL_TEMPLATE % ["alpha"]}, + "coreos-beta" => {box: "coreos-beta", user: "core", box_url: COREOS_URL_TEMPLATE % ["beta"]}, + "ubuntu1604" => {box: "generic/ubuntu1604", user: "vagrant"}, + "ubuntu1804" => {box: "generic/ubuntu1804", user: "vagrant"}, + "centos" => {box: "centos/7", user: "vagrant"}, + "centos-bento" => {box: "bento/centos-7.6", user: "vagrant"}, + "fedora" => {box: "fedora/28-cloud-base", user: "vagrant"}, + "opensuse" => {box: "opensuse/openSUSE-15.0-x86_64", user: "vagrant"}, + "opensuse-tumbleweed" => {box: "opensuse/openSUSE-Tumbleweed-x86_64", user: "vagrant"}, + "oraclelinux" => {box: "generic/oracle7", user: "vagrant"}, +} + +# Defaults for config options defined in CONFIG +$num_instances = 3 +$instance_name_prefix = "k8s" +$vm_gui = false +$vm_memory = 2048 +$vm_cpus = 1 +$shared_folders = {} +$forwarded_ports = {} +$subnet = "172.17.8" +$os = "centos" +$network_plugin = "flannel" +# Setting multi_networking to true will install Multus: https://github.com/intel/multus-cni +$multi_networking = false +# The first three nodes are etcd servers +$etcd_instances = $num_instances +# The first two nodes are kube masters +$kube_master_instances = $num_instances == 1 ? $num_instances : ($num_instances - 1) +# All nodes are kube nodes +$kube_node_instances = $num_instances +# The following only works when using the libvirt provider +$kube_node_instances_with_disks = false +$kube_node_instances_with_disks_size = "20G" +$kube_node_instances_with_disks_number = 2 +$override_disk_size = false +$disk_size = "20GB" +$local_path_provisioner_enabled = false +$local_path_provisioner_claim_root = "/opt/local-path-provisioner/" + +$playbook = "cluster.yml" + +host_vars = {} + +if File.exist?(CONFIG) + require CONFIG +end + +$box = SUPPORTED_OS[$os][:box] +# if $inventory is not set, try to use example +$inventory = "inventory/sample" if ! $inventory +$inventory = File.absolute_path($inventory, File.dirname(__FILE__)) + +# if $inventory has a hosts.ini file use it, otherwise copy over +# vars etc to where vagrant expects dynamic inventory to be +if ! File.exist?(File.join(File.dirname($inventory), "hosts.ini")) + $vagrant_ansible = File.join(File.dirname(__FILE__), ".vagrant", "provisioners", "ansible") + FileUtils.mkdir_p($vagrant_ansible) if ! File.exist?($vagrant_ansible) + if ! File.exist?(File.join($vagrant_ansible,"inventory")) + FileUtils.ln_s($inventory, File.join($vagrant_ansible,"inventory")) + end +end + +if Vagrant.has_plugin?("vagrant-proxyconf") + $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost" + (1..$num_instances).each do |i| + $no_proxy += ",#{$subnet}.#{i+100}" + end +end + +Vagrant.configure("2") do |config| + + config.vm.box = $box + if SUPPORTED_OS[$os].has_key? :box_url + config.vm.box_url = SUPPORTED_OS[$os][:box_url] + end + config.ssh.username = SUPPORTED_OS[$os][:user] + + # plugin conflict + if Vagrant.has_plugin?("vagrant-vbguest") then + config.vbguest.auto_update = false + end + + # always use Vagrants insecure key + config.ssh.insert_key = false + + if ($override_disk_size) + unless Vagrant.has_plugin?("vagrant-disksize") + system "vagrant plugin install vagrant-disksize" + end + config.disksize.size = $disk_size + end + + (1..$num_instances).each do |i| + config.vm.define vm_name = "%s-%01d" % [$instance_name_prefix, i] do |node| + + node.vm.hostname = vm_name + + if Vagrant.has_plugin?("vagrant-proxyconf") + node.proxy.http = ENV['HTTP_PROXY'] || ENV['http_proxy'] || "" + node.proxy.https = ENV['HTTPS_PROXY'] || ENV['https_proxy'] || "" + node.proxy.no_proxy = $no_proxy + end + + ["vmware_fusion", "vmware_workstation"].each do |vmware| + node.vm.provider vmware do |v| + v.vmx['memsize'] = $vm_memory + v.vmx['numvcpus'] = $vm_cpus + end + end + + node.vm.provider :virtualbox do |vb| + vb.memory = $vm_memory + vb.cpus = $vm_cpus + vb.gui = $vm_gui + vb.linked_clone = true + vb.customize ["modifyvm", :id, "--vram", "8"] # ubuntu defaults to 256 MB which is a waste of precious RAM + end + + node.vm.provider :libvirt do |lv| + lv.memory = $vm_memory + lv.cpus = $vm_cpus + lv.default_prefix = 'kubespray' + # Fix kernel panic on fedora 28 + if $os == "fedora" + lv.cpu_mode = "host-passthrough" + end + end + + if $kube_node_instances_with_disks + # Libvirt + driverletters = ('a'..'z').to_a + node.vm.provider :libvirt do |lv| + # always make /dev/sd{a/b/c} so that CI can ensure that + # virtualbox and libvirt will have the same devices to use for OSDs + (1..$kube_node_instances_with_disks_number).each do |d| + lv.storage :file, :device => "hd#{driverletters[d]}", :path => "disk-#{i}-#{d}-#{DISK_UUID}.disk", :size => $kube_node_instances_with_disks_size, :bus => "ide" + end + end + end + + if $expose_docker_tcp + node.vm.network "forwarded_port", guest: 2375, host: ($expose_docker_tcp + i - 1), auto_correct: true + end + + $forwarded_ports.each do |guest, host| + node.vm.network "forwarded_port", guest: guest, host: host, auto_correct: true + end + + node.vm.synced_folder ".", "/vagrant", disabled: false, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z'] , rsync__exclude: ['.git','venv'] + node.vm.synced_folder "../../", "/bigtop", disabled: false, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z'] , rsync__exclude: ['.git','venv'] + + $shared_folders.each do |src, dst| + node.vm.synced_folder src, dst, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z'] + end + + ip = "#{$subnet}.#{i+100}" + node.vm.network :private_network, ip: ip + + # Disable swap for each vm + node.vm.provision "shell", inline: "swapoff -a" + + host_vars[vm_name] = { + "ip": ip, + "flannel_interface": "eth1", + "kube_network_plugin": $network_plugin, + "kube_network_plugin_multus": $multi_networking, + "download_run_once": "True", + "download_localhost": "False", + "download_cache_dir": ENV['HOME'] + "/kubespray_cache", + # Make kubespray cache even when download_run_once is false + "download_force_cache": "True", + # Keeping the cache on the nodes can improve provisioning speed while debugging kubespray + "download_keep_remote_cache": "False", + "docker_keepcache": "1", + # These two settings will put kubectl and admin.config in $inventory/artifacts + "kubeconfig_localhost": "True", + "kubectl_localhost": "True", + "local_path_provisioner_enabled": "#{$local_path_provisioner_enabled}", + "local_path_provisioner_claim_root": "#{$local_path_provisioner_claim_root}", + "ansible_ssh_user": SUPPORTED_OS[$os][:user] + } + + # Only execute the Ansible provisioner once, when all the machines are up and ready. + if i == $num_instances + node.vm.provision "ansible" do |ansible| + ansible.playbook = $playbook + $ansible_inventory_path = File.join( $inventory, "hosts.ini") + if File.exist?($ansible_inventory_path) + ansible.inventory_path = $ansible_inventory_path + end + ansible.become = true + ansible.limit = "all" + ansible.host_key_checking = false + ansible.raw_arguments = ["--forks=#{$num_instances}", "--flush-cache", "-e ansible_become_pass=vagrant"] + ansible.host_vars = host_vars + #ansible.tags = ['download'] + ansible.groups = { + "etcd" => ["#{$instance_name_prefix}-[1:#{$etcd_instances}]"], + "kube-master" => ["#{$instance_name_prefix}-[1:#{$kube_master_instances}]"], + "kube-node" => ["#{$instance_name_prefix}-[1:#{$kube_node_instances}]"], + "k8s-cluster:children" => ["kube-master", "kube-node"], + } + end + end + + end + end +end