This is an automated email from the ASF dual-hosted git repository. marcoabreu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-mxnet-ci.git
The following commit(s) were added to refs/heads/master by this push: new 12de57a Add jenkins master (#7) 12de57a is described below commit 12de57a192f462e07446986b24bd2bd03edb06f6 Author: Marco de Abreu <marcoab...@users.noreply.github.com> AuthorDate: Fri Aug 16 00:03:34 2019 +0200 Add jenkins master (#7) --- services/jenkins-master/.gitignore | 6 + services/jenkins-master/README.md | 35 ++ services/jenkins-master/deploy.sh | 23 ++ services/jenkins-master/download_config.sh | 23 ++ services/jenkins-master/infrastructure.tf | 351 +++++++++++++++++++++ services/jenkins-master/prod/infrastructure.tfvars | 33 ++ .../jenkins-master/prod/jenkins/REDACTED-FULLY | 0 .../scripts/deploy_infrastructure.py | 186 +++++++++++ services/jenkins-master/scripts/docker_setup.sh | 163 ++++++++++ .../scripts/jenkins_config_templating.py | 275 ++++++++++++++++ services/jenkins-master/scripts/jenkins_setup.sh | 120 +++++++ .../jenkins-master/scripts/jenkins_sync_config.py | 134 ++++++++ services/jenkins-master/scripts/sync_ci_to_host.sh | 48 +++ services/jenkins-master/scripts/sync_host_to_ci.sh | 54 ++++ services/jenkins-master/test/infrastructure.tfvars | 39 +++ .../test/infrastructure_backend.tfvars | 22 ++ .../jenkins-master/test/jenkins/REDACYED-FULLY | 0 .../jenkins-master/test/jenkins_config.symlinkfile | 55 ++++ .../jenkins-master/test/jenkins_config.varfile | 8 + .../jenkins-master/test/secrets/REDACTED-FULLY | 0 services/jenkins-master/test/variables.sh | 21 ++ 21 files changed, 1596 insertions(+) diff --git a/services/jenkins-master/.gitignore b/services/jenkins-master/.gitignore new file mode 100644 index 0000000..fda640c --- /dev/null +++ b/services/jenkins-master/.gitignore @@ -0,0 +1,6 @@ +.terraform/ +jenkins.tar.bz2 +.terraform.tfstate.lock.info +__pycache__/ +jenkins_plugins.tar.bz2 +temp/ diff --git a/services/jenkins-master/README.md b/services/jenkins-master/README.md new file mode 100644 index 0000000..631da0b --- /dev/null +++ b/services/jenkins-master/README.md @@ -0,0 +1,35 @@ +<!--- Licensed to the Apache Software Foundation (ASF) under one --> +<!--- or more contributor license agreements. See the NOTICE file --> +<!--- distributed with this work for additional information --> +<!--- regarding copyright ownership. The ASF licenses this file --> +<!--- to you under the Apache License, Version 2.0 (the --> +<!--- "License"); you may not use this file except in compliance --> +<!--- with the License. You may obtain a copy of the License at --> + +<!--- http://www.apache.org/licenses/LICENSE-2.0 --> + +<!--- Unless required by applicable law or agreed to in writing, --> +<!--- software distributed under the License is distributed on an --> +<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --> +<!--- KIND, either express or implied. See the License for the --> +<!--- specific language governing permissions and limitations --> +<!--- under the License. --> + +This is the Terraform setup we used to create the Jenkins master. This script is **VERY** outdated! + +# Create infrastructure + +Warning: this will destroy the current DNS entries. + +``` +./init.sh +terraform init +terraform apply +``` + + +With difference instance type (overriding variables) + +``` +terraform apply -var instance_type=c1.xlarge +``` diff --git a/services/jenkins-master/deploy.sh b/services/jenkins-master/deploy.sh new file mode 100755 index 0000000..8ef0e04 --- /dev/null +++ b/services/jenkins-master/deploy.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +echo "Config dir:" +read config_dir + +python3 scripts/deploy_infrastructure.py --configdir "$config_dir" --terraformdir "." diff --git a/services/jenkins-master/download_config.sh b/services/jenkins-master/download_config.sh new file mode 100755 index 0000000..0d29ede --- /dev/null +++ b/services/jenkins-master/download_config.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +echo "Config dir:" +read config_dir + +python3 scripts/jenkins_sync_config.py --jenkinsdir "$config_dir/jenkins" --varfile "$config_dir/jenkins_config.varfile" --symlinkfile "$config_dir/jenkins_config.symlinkfile" --secretsdir "$config_dir/secrets" --tfvarsfile="$config_dir/infrastructure.tfvars" --mode "download" diff --git a/services/jenkins-master/infrastructure.tf b/services/jenkins-master/infrastructure.tf new file mode 100644 index 0000000..3bd16ed --- /dev/null +++ b/services/jenkins-master/infrastructure.tf @@ -0,0 +1,351 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +variable "key_name" { + type = "string" +} + +variable "key_path" { + type = "string" +} + +variable "instance_type" { + type = "string" +} + +variable "vpc_id" { + type = "string" +} + +variable "additional_security_group_ids" { + type = "list" +} + +variable "jenkins_config_bucket" { + type = "string" +} + +variable "zone_id" { + type = "string" +} + +variable "domain" { + type = "string" +} + +variable "shell_variables_file" { + type = "string" +} + +# AMI IDs can be retrieved at +# ftp://64.50.236.216/pub/ubuntu-cloud-images/query/xenial/server/released.txt +variable "ami" { + type = "string" + default = "ami-bd8f33c5" # Ubuntu 16.04 from 20180122 +} + +variable "instance_name" { + type = "string" +} + +variable "aws_availability_zone" { + type = "string" +} + +variable "aws_region" { + type = "string" +} + +variable "aws_access_key" { + type = "string" +} + +variable "aws_secret_key" { + type = "string" +} + +variable "ebs_volume_jenkins_master_state_volume_id" { + type = "string" +} + +provider "aws" { + access_key = "${var.aws_access_key}" + secret_key = "${var.aws_secret_key}" + region = "${var.aws_region}" +} + +# Store terraform state in S3 instead of local. +terraform { + backend "s3" { + key = "terraform.tfstate" + + # TODO: Lock statefile using dynamo db to prevent overriding statefiles if multiple + # people run terraform at the same time + # dynamodb_table = "terraform-state-${var.jenkins_config_bucket}-dynamo" + + # Remaining config is defined in $CONFIG_DIR/infrastructure_backend.tfvars + # See https://www.terraform.io/docs/backends/config.html for more details + } +} + +data "template_cloudinit_config" "user_data" { + # gzip = true + base64_encode = true + + # Important: This part has to be in first place as it gets mapped to + # /var/lib/cloud/instance/scripts/part-001 + # This is a hack, but there's no other way to reference other scripts + part { + content_type = "text/x-shellscript" + content = "${file("${var.shell_variables_file}")}" + } + + # part-002 + part { + content_type = "text/x-shellscript" + content = "${file("temp/jenkins_symlinks.sh")}" + } + + + # No Docker required on jenkins master + #part { + # content_type = "text/x-shellscript" + # content = "${file("scripts/docker_setup.sh")}" + #} + part { + content_type = "text/x-shellscript" + content = "${file("scripts/jenkins_setup.sh")}" + } +} + +# Require in order to allow attaching policies, so called trust relationship policy document +resource "aws_iam_role" "jenkins_master_role" { + name = "jenkins_master_role" + assume_role_policy = <<POLICY +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Principal": { + "Service": "ec2.amazonaws.com" + }, + "Effect": "Allow", + "Sid": "" + } + ] +} +POLICY +} + +resource "aws_iam_policy" "jenkins_master_s3_read_policy" { + name = "jenkins_master_s3_read_policy" + description = "Policy to grant Jenkins Master S3 read-access to the associated bucket" + policy = <<POLICY +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetBucketLocation", + "s3:ListAllMyBuckets" + ], + "Resource": "arn:aws:s3:::*" + }, + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::${var.jenkins_config_bucket}" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::${var.jenkins_config_bucket}/*" + ] + } + ] +} +POLICY +} + +resource "aws_iam_policy_attachment" "jenkins_master_s3_read_policy_attach" { + name = "jenkins_master_s3_read_policy_attach" + roles = ["${aws_iam_role.jenkins_master_role.name}"] + policy_arn = "${aws_iam_policy.jenkins_master_s3_read_policy.arn}" +} + +resource "aws_iam_instance_profile" "jenkins_master_profile" { + name = "jenkins_master_profile" + role = "${aws_iam_role.jenkins_master_role.name}" +} + +resource "aws_security_group" "allow_all_https" { + name = "tf_allow_all_https2" + description = "Allow all inbound traffic to https" + + ingress { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } +} + +resource "aws_security_group" "allow_all_www" { + name = "tf_allow_all_www2" + description = "Allow all inbound traffic to www" + + ingress { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } +} + + +resource "aws_instance" "mxnet-ci" { + # The connection block tells our provisioner how to + # communicate with the resource (instance) + connection { + # The default username for our AMI + user = "ubuntu" + + # The path to your keyfile + key_file = "${var.key_path}" + } + + # subnet ID for our VPC TODO + # subnet_id = "${var.vpc_id}" + # the instance type we want, comes from rundeck + instance_type = "${var.instance_type}" + + availability_zone = "${var.aws_availability_zone}" + + # Spot instance params: + #spot_price = "${var.spot_price}" + #wait_for_fulfillment = "true" + + # Lookup the correct AMI based on the region + # we specified + ami = "${var.ami}" + + iam_instance_profile = "${aws_iam_instance_profile.jenkins_master_profile.name}" + # The name of our SSH keypair you've created and downloaded + # from the AWS console. + # + # https://console.aws.amazon.com/ec2/v2/home?region=us-west-2#KeyPairs: + # + key_name = "${var.key_name}" + + #vpc_security_group_ids = "${var.security_groups_ids}" + + vpc_security_group_ids = ["${ + concat( + list(aws_security_group.allow_all_https.id), + list(aws_security_group.allow_all_www.id), + var.additional_security_group_ids + ) + }"] + user_data = "${data.template_cloudinit_config.user_data.rendered}" + + # We set the name as a tag. Not supported for spot instances: + # See https://github.com/hashicorp/terraform/issues/3263 + tags { + "Name" = "${var.instance_name}" + } + + + root_block_device { + volume_type = "gp2" + volume_size = 1000 + #iops = 15000 + delete_on_termination = true + } + + # Wait for S3 bucket as it's needed during startup + depends_on = [ + "aws_s3_bucket_object.jenkins_config_s3", + "aws_s3_bucket_object.jenkins_plugins_s3" + ] +} + +resource "aws_s3_bucket" "jenkins_config_bucket" { + bucket = "${var.jenkins_config_bucket}" + acl = "private" +} + +resource "aws_s3_bucket_object" "jenkins_config_s3" { + bucket = "${aws_s3_bucket.jenkins_config_bucket.id}" + key = "jenkins/jenkins.tar.bz2" + source = "temp/jenkins.tar.bz2" + etag = "${md5(file("temp/jenkins.tar.bz2"))}" +} + +resource "aws_s3_bucket_object" "jenkins_plugins_s3" { + bucket = "${aws_s3_bucket.jenkins_config_bucket.id}" + key = "jenkins/jenkins_plugins.tar.bz2" + source = "temp/jenkins_plugins.tar.bz2" + etag = "${md5(file("temp/jenkins_plugins.tar.bz2"))}" +} + +# TODO: Check for race conditions. Just in case: +# https://github.com/hashicorp/terraform/issues/2740#issuecomment-288549352 +resource "aws_volume_attachment" "ebs_jenkins_master_state" { + device_name = "/dev/sdf" + volume_id = "${var.ebs_volume_jenkins_master_state_volume_id}" + instance_id = "${aws_instance.mxnet-ci.id}" +} + +#output "user_data" { +# value = "${data.template_cloudinit_config.user_data.rendered}" +#} + + +resource "aws_route53_record" "mxnet-ci" { + zone_id = "${var.zone_id}" + name = "jenkins.${var.domain}" + type = "A" + ttl = "60" + records = ["${aws_instance.mxnet-ci.public_ip}"] +} + +resource "aws_route53_record" "mxnet-ci-private" { + zone_id = "${var.zone_id}" + name = "jenkins-priv.${var.domain}" + type = "A" + ttl = "60" + records = ["${aws_instance.mxnet-ci.private_ip}"] +} + +output "address" { + value = "${aws_instance.mxnet-ci.public_dns}" +} + +output "mxnet-ci" { + value = "${aws_route53_record.mxnet-ci.fqdn}" +} diff --git a/services/jenkins-master/prod/infrastructure.tfvars b/services/jenkins-master/prod/infrastructure.tfvars new file mode 100644 index 0000000..e34ab07 --- /dev/null +++ b/services/jenkins-master/prod/infrastructure.tfvars @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# TODO: NOT COMPLETE! +key_name = "REDACTED" +key_path = "~/.ssh/REDACTED" +instance_type = "c5.4xlarge" +vpc_id = "REDACTED" + +additional_security_group_ids = [ + "sg-d3dacfae", # VPC default + "sg-REDACTED" # REDACTED +] + +bucket = "mxnet-ci-master" +zone_id = "REDACTED" +domain = "mxnet-ci.amazon-ml.com" +instance_name = "MXNet-CI-Master" +aws_region = "us-west-2" diff --git a/services/jenkins-master/prod/jenkins/REDACTED-FULLY b/services/jenkins-master/prod/jenkins/REDACTED-FULLY new file mode 100644 index 0000000..e69de29 diff --git a/services/jenkins-master/scripts/deploy_infrastructure.py b/services/jenkins-master/scripts/deploy_infrastructure.py new file mode 100644 index 0000000..2bfc02e --- /dev/null +++ b/services/jenkins-master/scripts/deploy_infrastructure.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# -*- coding: utf-8 -*- + +# This script automates the Jenkins master deployment process. + +import argparse +import logging +import os +import re +import shutil +import subprocess +import tarfile +from shutil import copytree +from tempfile import TemporaryDirectory + +from jenkins_config_templating import execute_config_templating, assemble_symlink_list + +JENKINS_DIR_NAME = 'jenkins' +SECRET_DIR_NAME = 'secrets' +VARFILE_FILE_NAME = 'jenkins_config.varfile' +SYMLINKFILE_FILE_NAME = 'jenkins_config.symlinkfile' + +TERRAFORM_DEPLOY_TEMP_DIR = 'temp' +TERRAFORM_SCRIPT_NAME = 'infrastructure.tf' +TERRAFORM_VARFILE_NAME = 'infrastructure.tfvars' +TERRAFORM_BACKEND_VARFILE_NAME = 'infrastructure_backend.tfvars' + +JENKINS_PLUGINS_DIR_NAME = 'plugins' +JENKINS_CONFIG_TAR_NAME = 'jenkins.tar.bz2' +JENKINS_PLUGINS_TAR_NAME = 'jenkins_plugins.tar.bz2' +JENKINS_SYMLINK_FILE_NAME = 'jenkins_symlinks.sh' + +STATE_TOUCH_FILE_TEMPLATE = 'touch /ebs_jenkins_state/{} \n' +STATE_CREATE_DIR_TEMPLATE = 'mkdir -p /ebs_jenkins_state/{} \n' +STATE_SYMLINK_TEMPLATE = 'mkdir -p /var/lib/jenkins/{} && sudo ln -s /ebs_jenkins_state/{} /var/lib/jenkins/{} \n' + + +def main(): + logging.getLogger().setLevel(logging.DEBUG) + + parser = argparse.ArgumentParser() + parser.add_argument('-d', '--configdir', + help='Jenkins deployment configuration directory', + default='test', + type=str) + + parser.add_argument('-tf', '--terraformdir', + help='Directory containing the terraform scripts', + default='.', + type=str) + + args = parser.parse_args() + + terraform_dir_abs = os.path.abspath(args.terraformdir) + + if not os.path.isfile(os.path.join(terraform_dir_abs, TERRAFORM_SCRIPT_NAME)): + raise FileNotFoundError('Unable to find terraform script. Did you specify "--terraformdir"? {}'. + format(os.path.join(terraform_dir_abs, TERRAFORM_SCRIPT_NAME))) + + # Copy configuration to temp dir + with TemporaryDirectory() as temp_dir: + terraform_deploy_dir = os.path.join(terraform_dir_abs, TERRAFORM_DEPLOY_TEMP_DIR) + # Create deployment temp dir + if not os.path.exists(terraform_deploy_dir): + os.makedirs(terraform_deploy_dir) + + jenkins_dir = os.path.join(args.configdir, JENKINS_DIR_NAME) + temp_jenkins_dir = os.path.join(temp_dir, JENKINS_DIR_NAME) + logging.debug('Copying jenkins dir from {} to {}'.format(jenkins_dir, temp_jenkins_dir)) + copytree(jenkins_dir, temp_jenkins_dir, True) + + # Replace placeholders with actual secrets + execute_config_templating(os.path.join(args.configdir, VARFILE_FILE_NAME), + os.path.join(args.configdir, SECRET_DIR_NAME), temp_jenkins_dir, 'insert', + update_secrets=False) + logging.debug('Config replaced. Result can be found at {}'.format(temp_jenkins_dir)) + + # Assemble list of symlinks to be created + symlinks = assemble_symlink_list(os.path.join(args.configdir, SYMLINKFILE_FILE_NAME), temp_jenkins_dir) + + # Create shell script to symlink during startup + _create_symlink_shellscript(symlinks, os.path.join(terraform_deploy_dir, JENKINS_SYMLINK_FILE_NAME)) + + # Sanity: Ensure no state is part of config + _validate_config_contain_no_state(symlinks, temp_jenkins_dir) + + # Optional: Create backup of EBS + # TODO + + # Compress jenkins dir to allow upload to S3 + temp_jenkins_compressed_file = os.path.join(temp_dir, 'jenkins_config.tar.bz2') + temp_jenkins_compressed_plugins_file = os.path.join(temp_dir, 'jenkins_plugins.tar.bz2') + with tarfile.open(temp_jenkins_compressed_file, "w:bz2") as tar: + with tarfile.open(temp_jenkins_compressed_plugins_file, "w:bz2") as tar_plugin: + for file in os.listdir(temp_jenkins_dir): + logging.debug('Archiving {}'.format(file)) + # Since jenkins plugins are a few hundreds of megabytes, store them in a second compressed archive + # in order to prevent uploading (~15mins) them every single time the actual configuration is changed + if file != JENKINS_PLUGINS_DIR_NAME: + tar.add(os.path.join(temp_jenkins_dir, file), arcname=os.path.basename(file)) + else: + tar_plugin.add(os.path.join(temp_jenkins_dir, file), arcname=os.path.basename(file)) + + logging.info('Copying archives to {}'.format(terraform_dir_abs)) + # Copy generated archives to original dir + shutil.copy2(temp_jenkins_compressed_file, os.path.join(terraform_deploy_dir, JENKINS_CONFIG_TAR_NAME)) + shutil.copy2(temp_jenkins_compressed_plugins_file, os.path.join(terraform_deploy_dir, JENKINS_PLUGINS_TAR_NAME)) + + # Trigger terraform + logging.info('Running terraform...') + logging.debug('Switching current work dir to {}'.format(terraform_dir_abs)) + os.chdir(terraform_dir_abs) + + # Setting up the terraform S3 backend requires to have AWS credentials in the env vars - it's not able + # to access the variables file due interpolation in terraform being enabled after initialization of + # the s3 backend + env_vars = os.environ.copy() + env_vars['AWS_ACCESS_KEY_ID'] = _get_tfvars_entry(os.path.join(args.configdir, TERRAFORM_VARFILE_NAME), + 'aws_access_key') + env_vars['AWS_SECRET_ACCESS_KEY'] = _get_tfvars_entry(os.path.join(args.configdir, TERRAFORM_VARFILE_NAME), + 'aws_secret_key') + + p1 = subprocess.Popen('~/bin/terraform init -backend-config={}'. + format(os.path.join(args.configdir, TERRAFORM_BACKEND_VARFILE_NAME)), cwd=terraform_dir_abs, + env=env_vars, shell=True) + p1.wait() + p2 = subprocess.Popen('~/bin/terraform apply -var-file="{}"'. + format(os.path.join(args.configdir, TERRAFORM_VARFILE_NAME)), cwd=terraform_dir_abs, + env=env_vars, shell=True) + p2.wait() + logging.info('Deployment finished') + + +def _get_tfvars_entry(tfvars_file, key): + # This is just a hack because I don't want to spend the time to write an entire parser for the .tfvars format + with open(tfvars_file, 'r') as fp: + for line in fp: + if line.startswith(key): + result = re.search('"(.*)"', line).group(1) + return result + + raise ValueError('Could not find {} in {}'.format(key, tfvars_file)) + + +def _create_symlink_shellscript(symlinks, target_file): + with open(target_file, 'w') as fp: + for symlink_entry in symlinks: + # Ensure dirs and files exist on EBS before creating symlink + if symlink_entry.is_dir: + fp.write(STATE_CREATE_DIR_TEMPLATE.format(symlink_entry.filepath)) + else: + fp.write(STATE_TOUCH_FILE_TEMPLATE.format(symlink_entry.filepath)) + + # Create symlink + fp.write(STATE_SYMLINK_TEMPLATE. + format(os.path.dirname(symlink_entry.filepath), symlink_entry.filepath, symlink_entry.filepath)) + + +def _validate_config_contain_no_state(symlinks, jenkins_config_dir): + for symlink_entry in symlinks: + path = os.path.join(jenkins_config_dir, symlink_entry.filepath) + if os.path.isfile(path) or os.path.isdir(path): + raise FileExistsError( + '{} is defined as state, but included in config. Remove before continuing.'.format(path)) + + +if __name__ == '__main__': + main() diff --git a/services/jenkins-master/scripts/docker_setup.sh b/services/jenkins-master/scripts/docker_setup.sh new file mode 100755 index 0000000..87d9f38 --- /dev/null +++ b/services/jenkins-master/scripts/docker_setup.sh @@ -0,0 +1,163 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# To verify if these steps execute correctly, you can check /var/log/cloud-init-output.log in the instances +# Cloud init doesn't log anymore, so we log to user-data log and syslog + +# UserData script which installs docker and code deploy agent in AML / Ubuntu + +set -e +set -x + +#Just log to cloud-init-output.log +#exec > >(tee -a /var/log/user-data.log|logger -t user-data ) 2>&1 + +DISTRO=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +DISTRO=${DISTRO//\"/} + +echo "Running on $DISTRO" + +function install_docker_ubuntu() { + #apt-get -y install docker docker.io + export DEBIAN_FRONTEND=noninteractive + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + apt-get update + apt-get -y install docker-ce + service docker restart + usermod -a -G docker ubuntu +} + +function install_docker_aml() { + yum -y install docker.x86_64 + sudo usermod -a -G docker ec2-user + service docker restart +} + +function install_code_deploy_agent() { + # Install code deploy agent + pushd . + TMP_INSTALL=/tmp/code_deploy + mkdir -p $TMP_INSTALL + cd $TMP_INSTALL + wget https://aws-codedeploy-eu-central-1.s3.amazonaws.com/latest/install + chmod +x ./install + ./install auto + popd + rm -rf $TMP_INSTALL +} + + + +case $DISTRO in +"Ubuntu") + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get -y install ruby wget vim-nox silversearcher-ag htop + install_code_deploy_agent + install_docker_ubuntu + ;; +"Amazon Linux AMI") + echo "AL" + install_docker_aml + install_code_deploy_agent + yum -y install git python35 + ;; +*) + echo "?" + ;; +esac + + + + + +# Use the ephemeral storage for docker so we don't run out of space +function setup_docker_ephemeral() { + if mountpoint -q -- /mnt; then + echo "Using /mnt/docker for docker storage" + service docker stop + mkdir -p /mnt/docker + mount -o bind /mnt/docker/ /var/lib/docker/ + service docker start + fi +} + + +function setup_ephemeral_raid() { + METADATA_URL_BASE="http://169.254.169.254/2016-09-02/" + DRIVE_SCHEME=`mount | perl -ne 'if(m#/dev/(xvd|sd).\d?#) { print "$1\n"; exit}'` + + drives="" + ephemeral_count=0 + ephemerals=$(curl --silent $METADATA_URL_BASE/meta-data/block-device-mapping/ | grep ephemeral) + for e in $ephemerals; do + echo "Probing $e .." + device_name=$(curl --silent $METADATA_URL_BASE/meta-data/block-device-mapping/$e) + # might have to convert 'sdb' -> 'xvdb' + device_name=$(echo $device_name | sed "s/sd/$DRIVE_SCHEME/") + device_path="/dev/$device_name" + + # test that the device actually exists since you can request more ephemeral drives than are available + # for an instance type and the meta-data API will happily tell you it exists when it really does not. + if [ -b $device_path ]; then + echo "Detected ephemeral disk: $device_path" + drives="$drives $device_path" + ephemeral_count=$((ephemeral_count + 1 )) + umount $device_path || true + else + echo "Ephemeral disk $e, $device_path is not present. skipping" + fi + done + + if [ "$ephemeral_count" = 0 ]; then + echo "No ephemeral disk detected." + return 1 + fi + + # overwrite first few blocks in case there is a filesystem, otherwise mdadm will prompt for input + for drive in $drives; do + dd if=/dev/zero of=$drive bs=4096 count=1024 + done + + partprobe || true + # Force in case there's only one drive + mdadm --create --force --verbose /dev/md0 --level=0 -c256 --raid-devices=$ephemeral_count $drives + echo DEVICE $drives | tee /etc/mdadm.conf + mdadm --detail --scan | tee -a /etc/mdadm.conf + blockdev --setra 65536 /dev/md0 + mkfs -t ext4 -m 0 /dev/md0 + mount -t ext4 -o noatime /dev/md0 /mnt + + # Remove xvdb/sdb from fstab + chmod 777 /etc/fstab + sed -i "/${DRIVE_SCHEME}b/d" /etc/fstab + + # Make raid appear on reboot + echo "/dev/md0 /mnt ext4 noatime 0 0" | tee -a /etc/fstab + return 0 +} + +# Add your custom initialization code below +(setup_ephemeral_raid && setup_docker_ephemeral) || true + +echo "UserData initialization is done" diff --git a/services/jenkins-master/scripts/jenkins_config_templating.py b/services/jenkins-master/scripts/jenkins_config_templating.py new file mode 100644 index 0000000..abf8c79 --- /dev/null +++ b/services/jenkins-master/scripts/jenkins_config_templating.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# -*- coding: utf-8 -*- + +# This script serves the purpose to replace sensitive parts in the jenkins configuration with placeholders in order +# to allow the configuration to be published to a public repository + +import argparse +import filecmp +import glob +import json +import logging +import os +import pathlib +import shutil +from collections import namedtuple + +from lxml import etree + +SECRET_ENTRY_KEYS = ['filepath', 'xpath', 'secret', 'placeholder'] +SecretEntry = namedtuple('SecretEntry', SECRET_ENTRY_KEYS) + +SYMLINK_ENTRY_KEYS = ['filepath', 'is_dir'] +SymlinkEntry = namedtuple('SymlinkEntry', SYMLINK_ENTRY_KEYS) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-vf', '--varfile', + help='Location of the variable file', + type=str) + + parser.add_argument('-sf', '--symlinkfile', + help='Location of the symlink file', + type=str) + + parser.add_argument('-sd', '--secretsdir', + help='Location of the directory containing secrets', + type=str) + + parser.add_argument('-jd', '--jenkinsdir', + help='Location of the jenkins directory', + type=str) + + parser.add_argument('-m', '--mode', + help='"remove" or "insert" credentials', + default='insert', + type=str) + + args = parser.parse_args() + + execute_config_templating(args.varfile, args.secretsdir, args.jenkinsdir, args.mode, update_secrets=False) + # TODO: Add symlink list creation + + +def execute_config_templating(varfile, secretsdir, jenkinsdir, mode, update_secrets): + """ + Execute config templating that inserts or removes secrets from a jenkins configuration directory + :param varfile: File containing the actual variables that should be used during replacement + :param secretsdir: Directory containing all files that should be just copied or removed as secrets + :param jenkinsdir: Jenkins configuration directory + :param mode: 'insert' replaces placeholders with actual values. 'remove' removes these values and + inserts placeholders + :param update_secrets: Update secrets if mode == 'remove' and actual secrets differ from the stored ones + :return: + """ + secret_entries = read_secret_entires(varfile) + logging.debug('Found {} secret entries to be replaced'.format(len(secret_entries))) + + # Prepare by finding all unique identifiers. This is required because XML parsers do not allow in-place + # replacements. Instead, we're reading a unique identifier under the specified xpath and then try to + # replace it in-place by using search-and-replace. This method will be aborted if the current value + # has been found multiple times within the same file. + for root, dirs, files in os.walk(secretsdir, topdown=False): + for name in files: + original_path = os.path.join(root, name) + rel_path = os.path.relpath(original_path, secretsdir) + temp_path = os.path.join(jenkinsdir, rel_path) + + if mode == 'insert': + pathlib.Path(os.path.dirname(temp_path)).mkdir(parents=True, exist_ok=True) + shutil.copyfile(original_path, temp_path) + elif mode == 'remove': + # Check if secret does not exist anymore + if not os.path.isfile(temp_path): + if update_secrets: + logging.info('Deleting secret {} because it has been removed on target'.format(rel_path)) + os.remove(original_path) + else: + raise ValueError('Secret {} has been deleted'.format(rel_path)) + + # Check if secrets are the same or have to be updated + if not filecmp.cmp(temp_path, original_path): + if update_secrets: + logging.info('Replacing secret {} due to changed content'.format(rel_path)) + shutil.copyfile(temp_path, original_path) + else: + raise ValueError('Secret {} contains changed content'.format(rel_path)) + + os.remove(temp_path) + else: + raise ValueError('Mode {} unknown'.format(mode)) # TODO check this previously + + # Check if any files in the secrets-dir are left that didn't exist in the previous config. Unfortunately, + # we can't verify if secrets outside the secrets-dir have been added. + if mode == 'remove': + temp_secrets_dir = os.path.join(jenkinsdir, 'secrets') + for root, dirs, files in os.walk(temp_secrets_dir, topdown=False): + for name in files: + temp_path = os.path.join(root, name) + rel_path = os.path.relpath(temp_path, jenkinsdir) + original_path = os.path.join(secretsdir, rel_path) + + if update_secrets: + logging.info('Adding new secret at {}'.format(rel_path)) + shutil.copyfile(temp_path, original_path) + else: + raise ValueError('New secret at {}'.format(rel_path)) + shutil.rmtree(temp_secrets_dir) + + for secret_entry in secret_entries: + temp_file_path = os.path.join(jenkinsdir, secret_entry.filepath) + if os.path.isfile(temp_file_path): + element = etree.parse(temp_file_path).xpath(secret_entry.xpath) + + # Check if xpath delivers multiple results. The xpath should only match once + if len(element) == 1: + current_value = element[0].text + if not current_value.strip(): + raise ValueError('Element {} at {}:{} is not a text field'. + format(current_value, temp_file_path, secret_entry.xpath)) + + if mode == 'insert': + expected_value = secret_entry.placeholder + target_value = secret_entry.secret + elif mode == 'remove': + expected_value = secret_entry.secret + target_value = secret_entry.placeholder + else: + raise ValueError('Mode {} unknown'.format(mode)) + + if current_value == expected_value: + logging.debug( + 'Replacing {} with {} at {}:{}'.format(current_value, target_value, secret_entry.xpath, + temp_file_path)) + _replace_values(current_value, target_value, temp_file_path) + + elif current_value == target_value: + logging.info('Target value {} already present. Skipping {}:{}'. + format(target_value, secret_entry.xpath, temp_file_path)) + continue + else: + raise ValueError('Current value "{}" does not match expected value "{}" in {}:{}'.format( + current_value, expected_value, secret_entry.xpath)) + elif len(element) == 0: + raise ValueError('Element at {}:{} not found'.format(temp_file_path, secret_entry.xpath)) + else: + raise ValueError('1 Element expected, {} found at {}:{}'. + format(len(element), temp_file_path, secret_entry.xpath)) + else: + raise FileNotFoundError('Could not find file {}'.format(temp_file_path)) + + +def assemble_symlink_list(symlink_file, jenkinsdir): + """ + Assemble a list of files that should be symlinked during startup, providing support for state files on EBS + :param symlink_file: File containing path expressions to describe the symlinked files and dirs + :param jenkinsdir: Jenkins configuration directory + :return: Array of SymlinkEntry + """ + symlink_config = read_symlink_entries(symlink_file) + logging.debug('Found {} symlink entries'.format(len(symlink_config))) + + symlinks = [] + + for symlink_entry in symlink_config: + input_path_expression = os.path.join(jenkinsdir, symlink_entry.filepath) + result_paths = [] + + if '*' in symlink_entry.filepath: + # If path contains a wildcard, search for all results + wildcard_path_split = input_path_expression.split('*') + if len(wildcard_path_split) == 1: + result_paths = glob.glob(input_path_expression) + elif len(wildcard_path_split) == 2: + # If wildcard is in the middle and target directories don't exist, result would be empty. + # Instead, iterate manually + for partial_path in glob.glob(os.path.join(wildcard_path_split[0], '*')): + result_paths.append(os.path.join(partial_path, wildcard_path_split[1].lstrip('/'))) + else: + raise ValueError('Symlink expression may only contain one wildcard') + logging.debug('Resolving {} to {}'.format(symlink_entry.filepath, result_paths)) + else: + result_paths = [input_path_expression] + + for abs_path in result_paths: + rel_path = os.path.relpath(abs_path, jenkinsdir) + symlinks.append(SymlinkEntry(rel_path, symlink_entry.is_dir)) + + return symlinks + + +def read_secret_entires(varfile): + """ + Read SecretEntry from varfile + :param varfile: File containing SecretEntries as JSON + :return: Array of SecretEntry + """ + with open(varfile, 'r') as fp: + secret_dict_raw = json.load(fp) + secrets = [] + for secret_entry_dict in secret_dict_raw: + # Import values in the same order as expected in SecretEntry. Reason being that the values are expected + # to be inserted in the same order as defined previously. + secrets.append(SecretEntry(*[secret_entry_dict[k] for k in SECRET_ENTRY_KEYS])) + return secrets + + +def read_symlink_entries(varfile): + """ + Read SymlinkEntry from varfile + :param varfile: File containing SymlinkEntries as JSON + :return: Array of SymlinkEntries + """ + with open(varfile, 'r') as fp: + symlink_dict_raw = json.load(fp) + symlinks = [] + for symlink_entry_dict in symlink_dict_raw: + # Import values in the same order as expected in SecretEntry. Reason being that the values are expected + # to be inserted in the same order as defined previously. + symlinks.append(SymlinkEntry(*[symlink_entry_dict[k] for k in SYMLINK_ENTRY_KEYS])) + return symlinks + + +def _replace_values(current_value, target_value, temp_file_path): + # Only execute changes on the temp file + with open(temp_file_path, 'r+') as temp_fh: + temp_file_content = temp_fh.read() + + # Check if current value is unique within the temp file to prevent injection attacks + nb_occurrences_temp = temp_file_content.count(current_value) + if nb_occurrences_temp != 1: + raise ValueError( + '1 occurrence of current value {} in file {} expected, {} found'.format( + current_value, + temp_file_path, + nb_occurrences_temp)) + + temp_file_content_replaced = temp_file_content.replace(current_value, target_value) + + # Write replaced content back to the temp file + temp_fh.seek(0) + temp_fh.write(temp_file_content_replaced) + temp_fh.truncate() + + +if __name__ == '__main__': + main() diff --git a/services/jenkins-master/scripts/jenkins_setup.sh b/services/jenkins-master/scripts/jenkins_setup.sh new file mode 100755 index 0000000..393ff29 --- /dev/null +++ b/services/jenkins-master/scripts/jenkins_setup.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -x +set -e + +# Just log to cloud-init-output.log +# exec > >(tee -a /var/log/user-data.log|logger -t user-data ) 2>&1 + +export DEBIAN_FRONTEND=noninteractive + +######################### +# Import variables +source /var/lib/cloud/instance/scripts/part-001 + +# Validate that environment variables are set +if [ -z "$DNS_ADDRESS" ]; then + echo "Need to set DNS_ADDRESS" + exit 1 +fi + +if [ -z "$S3_MASTER_BUCKET" ]; then + echo "Need to set S3_MASTER_BUCKET" + exit 1 +fi + +######################### + +######################### +# Hostname +echo $DNS_ADDRESS > /etc/hostname +hostname $DNS_ADDRESS +echo 'search $DNS_ADDRESS' >> /etc/resolv.conf + +######################### +# Install jenkins. Use weekly instead of stable due to high number of security vulnerabilities +wget -q -O - https://pkg.jenkins.io/debian/jenkins.io.key | sudo apt-key add - +echo deb http://pkg.jenkins.io/debian binary/ > /etc/apt/sources.list.d/jenkins.list +apt-get update +apt-get -y install openjdk-8-jre-headless +apt-get -y install jenkins mailutils +service jenkins stop + +######################### +rm -rf /var/lib/jenkins +mkdir /var/lib/jenkins +chown -R jenkins.jenkins /var/lib/jenkins +######################### + +######################### +#Port 1-1000 are only acccessible as root. Jenkins does not run as root, thus port forwarding is +#required +network_interface=$(ip link | awk -F: '$0 !~ "lo|vir|wl|^[^0-9]"{print $2}') +iptables -t nat -I PREROUTING -p tcp -i $network_interface --dport 80 -j REDIRECT --to-ports 8080 +iptables -t nat -I PREROUTING -p tcp -i $network_interface --dport 443 -j REDIRECT --to-ports 8081 +apt-get -y install iptables-persistent +######################### + +######################### +# Attach Jenkins state EBS volume +mkdir /ebs_jenkins_state + +# Wait until volume mounted +until [ -e /dev/nvme1n1 ] +do + echo 'Waiting for /dev/nvme1n1 to be mounted' + sleep 1 +done + +mount /dev/nvme1n1 /ebs_jenkins_state/ + +# Ensure volume is mounted upon restart +sudo su root -c '(crontab -l 2>/dev/null; echo "@reboot /bin/mount /dev/nvme1n1 /ebs_jenkins_state") | crontab -' + +chown -R jenkins.jenkins /ebs_jenkins_state +######################### + +######################### +# Unpack preconfigured jenkins +apt-get -y install awscli +tmpdir=$(mktemp -d) +aws s3 cp --quiet s3://$S3_MASTER_BUCKET/jenkins/jenkins.tar.bz2 $tmpdir +aws s3 cp --quiet s3://$S3_MASTER_BUCKET/jenkins/jenkins_plugins.tar.bz2 $tmpdir + +# Softlink state and cache files +source /var/lib/cloud/instance/scripts/part-002 +chown -R jenkins.jenkins /ebs_jenkins_state + +# Copy preconfigured jenkins +tar -C /var/lib/jenkins -xjf $tmpdir/jenkins.tar.bz2 +tar -C /var/lib/jenkins -xjf $tmpdir/jenkins_plugins.tar.bz2 +chown -R jenkins.jenkins /var/lib/jenkins +######################### + + + +######################### +# Set jenkins arguments (enable https) +sed -i 's#JENKINS_ARGS.*#JENKINS_ARGS="--webroot=/var/cache/$NAME/war --httpPort=8080 --httpsPort=8081"#' /etc/default/jenkins +sed -i 's#JAVA_ARGS.*#JAVA_ARGS="-Xmx8192m -Xms512m -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -Djava.awt.headless=true"#' /etc/default/jenkins +######################### + +service jenkins start +echo "Jenkins setup completed successfully" diff --git a/services/jenkins-master/scripts/jenkins_sync_config.py b/services/jenkins-master/scripts/jenkins_sync_config.py new file mode 100644 index 0000000..4bd3740 --- /dev/null +++ b/services/jenkins-master/scripts/jenkins_sync_config.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# -*- coding: utf-8 -*- + +# This script synchronizes the local config with the running configuration on a jenkins master server + +import argparse +import glob +import os +import re +import shutil +from distutils.dir_util import copy_tree +from tempfile import TemporaryDirectory + +from jenkins_config_templating import execute_config_templating, read_symlink_entries + +BASH_SCRIPT_JENKINS_TO_TEMP = \ + 'ssh-keygen -R {}; ssh -C ubuntu@{} "bash -s" <<EOS \n' \ + 'sudo rm -rf /home/ubuntu/jenkins; mkdir -p /home/ubuntu/jenkins; \n' \ + 'sudo cp -RP --verbose /var/lib/jenkins/* /home/ubuntu/jenkins \n' \ + 'sudo chown -R ubuntu.ubuntu /home/ubuntu/jenkins; \n' \ + 'find /home/ubuntu/jenkins/ -type l -delete; \n' \ + 'EOS' + +BASH_SCRIPT_DOWNLOAD_TEMP = 'rsync --delete -zvaP ubuntu@{}:jenkins/ {}' +BASH_SCRIPT_SYNC_LOCAL = 'rsync --delete -zvaP {}/* {}' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-jd', '--jenkinsdir', + help='Location of the jenkins directory', + type=str) + + parser.add_argument('-vf', '--varfile', + help='Location of the variable file', + type=str) + + parser.add_argument('-sf', '--symlinkfile', + help='Location of the symlink file', + type=str) + + parser.add_argument('-sd', '--secretsdir', + help='Location of the directory containing secrets', + type=str) + + parser.add_argument('-tf', '--tfvarsfile', + help='Location of the terraform variable file', + type=str) + + parser.add_argument('-m', '--mode', + help='"download" or "upload" config', + type=str) + + args = parser.parse_args() + + jenkins_sync_config(args.mode, args.jenkinsdir, args.varfile, args.symlinkfile, args.secretsdir, args.tfvarsfile) + + +def jenkins_sync_config(mode, jenkins_dir, var_file, symlink_file, secrets_dir, tfvars_file): + # secret_entries = read_secret_entires(var_file) #TODO: Verify no new secrets have been downloaded + symlink_config = read_symlink_entries(symlink_file) + jenkins_address = 'jenkins.' + _get_tfvars_entry(tfvars_file, 'domain') + + with TemporaryDirectory() as temp_dir: + if mode == 'download': + # Copy config to temp dir on jenkins master to avoid permission issues due to owner being jenkins while + # rsync is logging in as ubuntu. We're using bash scripts instead of an SSH client because all python + # libraries for SSH usage are having trouble when we supply a custom rsa key instead ouf using id_rsa + # TODO: Use proper SSH client + bash_jenkins_to_temp_cmd = BASH_SCRIPT_JENKINS_TO_TEMP.format(jenkins_address, jenkins_address) + os.system(bash_jenkins_to_temp_cmd) + + # Copy old jenkins to local temp dir to allow rsync and thus speed up the download process due to diff + copy_tree(jenkins_dir, temp_dir) + bash_jenkins_download_cmd = BASH_SCRIPT_DOWNLOAD_TEMP.format(jenkins_address, temp_dir) + os.system(bash_jenkins_download_cmd) + + # Delete state files. Symlinks are already deleted before config is downloaded, but there might be new dirs + # which were not symlinked yet. + _delete_state_files(symlink_config, temp_dir) + + # Remove secrets according to secret config + execute_config_templating(var_file, secrets_dir, temp_dir, 'remove', update_secrets=True) + + # TODO Optional: Verify no new secrets have been downloaded + + # Move new config to configdir + bash_sync_local_cmd = BASH_SCRIPT_SYNC_LOCAL.format(temp_dir, jenkins_dir) + os.system(bash_sync_local_cmd) + else: + raise ValueError('Mode {} not supported'.format(mode)) + + +def _delete_state_files(symlink_config, jenkins_dir): + for symlink_entry in symlink_config: + result_paths = glob.glob(os.path.join(jenkins_dir, symlink_entry.filepath)) + for path in result_paths: + if symlink_entry.is_dir: + shutil.rmtree(path) + else: + os.remove(path) + + +def _get_tfvars_entry(tfvars_file, key): + # This is just a hack because I don't want to spend the time to write an entire parser for the .tfvars format + with open(tfvars_file, 'r') as fp: + for line in fp: + if line.startswith(key): + result = re.search('"(.*)"', line).group(1) + return result + + raise ValueError('Could not find {} in {}'.format(key, tfvars_file)) + + +if __name__ == '__main__': + main() diff --git a/services/jenkins-master/scripts/sync_ci_to_host.sh b/services/jenkins-master/scripts/sync_ci_to_host.sh new file mode 100755 index 0000000..3e290c5 --- /dev/null +++ b/services/jenkins-master/scripts/sync_ci_to_host.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# +# Sync modifications to jenkins configuration to local +# + + +set -e +set -x + +# Work around permissions of /var/lib/jenkins +ssh -C ubu...@jenkins.mxnet-ci.amazon-ml.com "bash -s" <<EOS +set -e +set -x +sudo su +mkdir -p /home/ubuntu/jenkins +rsync --delete --exclude="workspace/" --exclude="updates/" --exclude="logs/" --exclude=".cache/" --exclude="fingerprints/" --exclude="org.jenkinsci.plugins.github.GitHubPlugin.cache/" --exclude="builds/" -vaP /var/lib/jenkins/ /home/ubuntu/jenkins +chown -R ubuntu.ubuntu /home/ubuntu/jenkins +EOS + +rsync --delete -zvaP ubu...@jenkins.mxnet-ci.amazon-ml.com:jenkins/ jenkins/ +rm -rf jenkins/.cache +rm -rf jenkins/logs +rm -rf jenkins/fingerprints +rm -rf jenkins/org.jenkinsci.plugins.github.GitHubPlugin.cache/* + + + +ssh -C ubu...@jenkins.mxnet-ci.amazon-ml.com "bash -s" <<EOS +rm -rf /home/ubuntu/jenkins +EOS diff --git a/services/jenkins-master/scripts/sync_host_to_ci.sh b/services/jenkins-master/scripts/sync_host_to_ci.sh new file mode 100755 index 0000000..de7a354 --- /dev/null +++ b/services/jenkins-master/scripts/sync_host_to_ci.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# +# Sync modifications to jenkins configuration to local +# + +set -e +set -x +# Create temporary directory on the CI +ssh -C ubu...@jenkins.mxnet-ci.amazon-ml.com "bash -s" << EOS +set -e +set -x +sudo su +rm -rf /home/ubuntu/jenkins/ +mkdir /home/ubuntu/jenkins/ +chown ubuntu.ubuntu /home/ubuntu/jenkins/ +EOS + +# Copy host files to the temp CI dir +rsync -zvaP jenkins/ ubu...@jenkins.mxnet-ci.amazon-ml.com:jenkins/ + +# Stop running jenkins, preserve state-informations, deploy the changes and start Jenkins +ssh -C ubu...@jenkins.mxnet-ci.amazon-ml.com "bash -s" <<EOF +set -e +set -x +sudo su +service jenkins stop +mv -t /home/ubuntu/jenkins /var/lib/jenkins/workspace/ /var/lib/jenkins/updates/ +/var/lib/jenkins/logs/ +/var/lib/jenkins/.cache//var/lib/jenkins/fingerprints//var/lib/jenkins/org.jenkinsci.plugins.github.GitHubPlugin.cache/ +/var/lib/jenkins/builds/ +rsync -vaP /home/ubuntu/jenkins/ /var/lib/jenkins/ +rm -rf /home/ubuntu/jenkins/ +chown -R jenkins.jenkins /var/lib/jenkins/ +service jenkins start +EOF + diff --git a/services/jenkins-master/test/infrastructure.tfvars b/services/jenkins-master/test/infrastructure.tfvars new file mode 100644 index 0000000..989ef33 --- /dev/null +++ b/services/jenkins-master/test/infrastructure.tfvars @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +aws_access_key = "REDACTED" +aws_secret_key = "REDACTED" +key_name = "REDACTED" +key_path = "~/.ssh/REDACTED" +instance_type = "c5.4xlarge" +vpc_id = "02c00d7b" + +additional_security_group_ids = [ + "sg-5d83d421", # VPC default + "sg-REDACTED" # REDACTED +] + +shell_variables_file = "test/variables.sh" +jenkins_config_bucket = "mxnet-ci-master-dev" +zone_id = "REDACTED" +domain = "mxnet-ci-dev.amazon-ml.com" +instance_name = "MXNet-CI-Master" +aws_region = "us-west-2" + +# EBS volume and AZ have to be the same +aws_availability_zone = "us-west-2b" +ebs_volume_jenkins_master_state_volume_id = "vol-REDACTED" diff --git a/services/jenkins-master/test/infrastructure_backend.tfvars b/services/jenkins-master/test/infrastructure_backend.tfvars new file mode 100644 index 0000000..8ad16f3 --- /dev/null +++ b/services/jenkins-master/test/infrastructure_backend.tfvars @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Terraform backend configuration. See See https://www.terraform.io/docs/backends/config.html for +# more details. Reason being that config can not be interpolated during backend init + +bucket = "mxnet-ci-master-dev-tfstate" +region = "us-west-2" diff --git a/services/jenkins-master/test/jenkins/REDACYED-FULLY b/services/jenkins-master/test/jenkins/REDACYED-FULLY new file mode 100644 index 0000000..e69de29 diff --git a/services/jenkins-master/test/jenkins_config.symlinkfile b/services/jenkins-master/test/jenkins_config.symlinkfile new file mode 100644 index 0000000..80144ac --- /dev/null +++ b/services/jenkins-master/test/jenkins_config.symlinkfile @@ -0,0 +1,55 @@ +[ + { + "filepath": "caches", + "is_dir": 1 + }, + { + "filepath": "fingerprints", + "is_dir": 1 + }, + { + "filepath": "jobs/*/branches", + "is_dir": 1 + }, + { + "filepath": "jobs/*/builds", + "is_dir": 1 + }, + { + "filepath": "jobs/*/indexing", + "is_dir": 1 + }, + { + "filepath": "logs", + "is_dir": 1 + }, + { + "filepath": "users", + "is_dir": 1 + }, + { + "filepath": "workspace", + "is_dir": 1 + }, + { + "filepath": "updates", + "is_dir": 1 + }, + + { + "filepath": "org.jenkinsci.plugins.workflow.flow.FlowExecutionList.xml", + "is_dir": 0 + }, + { + "filepath": "queue.xml.bak", + "is_dir": 0 + }, + { + "filepath": "queue.xml", + "is_dir": 0 + }, + { + "filepath": "jobs/*/nextBuildNumber", + "is_dir": 0 + } +] diff --git a/services/jenkins-master/test/jenkins_config.varfile b/services/jenkins-master/test/jenkins_config.varfile new file mode 100644 index 0000000..c0c5885 --- /dev/null +++ b/services/jenkins-master/test/jenkins_config.varfile @@ -0,0 +1,8 @@ +[ + { + "filepath": "config.xml", + "placeholder": "[GitHubSecurityRealm/clientSecret]", + "secret": "REDACTED", + "xpath": "/hudson/securityRealm[@class='org.jenkinsci.plugins.GithubSecurityRealm']/clientSecret" + } +] diff --git a/services/jenkins-master/test/secrets/REDACTED-FULLY b/services/jenkins-master/test/secrets/REDACTED-FULLY new file mode 100644 index 0000000..e69de29 diff --git a/services/jenkins-master/test/variables.sh b/services/jenkins-master/test/variables.sh new file mode 100644 index 0000000..5fcc90c --- /dev/null +++ b/services/jenkins-master/test/variables.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export DNS_ADDRESS="mxnet-ci-dev.amazon-ml.com" +export S3_MASTER_BUCKET="mxnet-ci-master-dev"