This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch jenkins-master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet-ci.git
commit 0dddb7f9eedad5a85e29626ad1cb0f99fa78b7d7
Author: Marco de Abreu <marco.g.abreu+git...@gmail.com>
AuthorDate: Thu Aug 15 23:58:22 2019 +0200
Add jenkins master
---
services/jenkins-master/.gitignore | 6 +
services/jenkins-master/README.md | 35 ++
services/jenkins-master/deploy.sh | 23 ++
services/jenkins-master/download_config.sh | 23 ++
services/jenkins-master/infrastructure.tf | 351 +++++++++++++++++++++
services/jenkins-master/prod/infrastructure.tfvars | 33 ++
.../jenkins-master/prod/jenkins/REDACTED-FULLY | 0
.../scripts/deploy_infrastructure.py | 186 +++++++++++
services/jenkins-master/scripts/docker_setup.sh | 163 ++++++++++
.../scripts/jenkins_config_templating.py | 275 ++++++++++++++++
services/jenkins-master/scripts/jenkins_setup.sh | 120 +++++++
.../jenkins-master/scripts/jenkins_sync_config.py | 134 ++++++++
services/jenkins-master/scripts/sync_ci_to_host.sh | 48 +++
services/jenkins-master/scripts/sync_host_to_ci.sh | 54 ++++
services/jenkins-master/test/infrastructure.tfvars | 39 +++
.../test/infrastructure_backend.tfvars | 22 ++
.../jenkins-master/test/jenkins/REDACYED-FULLY | 0
.../jenkins-master/test/jenkins_config.symlinkfile | 55 ++++
.../jenkins-master/test/jenkins_config.varfile | 8 +
.../jenkins-master/test/secrets/REDACTED-FULLY | 0
services/jenkins-master/test/variables.sh | 21 ++
21 files changed, 1596 insertions(+)
diff --git a/services/jenkins-master/.gitignore
b/services/jenkins-master/.gitignore
new file mode 100644
index 0000000..fda640c
--- /dev/null
+++ b/services/jenkins-master/.gitignore
@@ -0,0 +1,6 @@
+.terraform/
+jenkins.tar.bz2
+.terraform.tfstate.lock.info
+__pycache__/
+jenkins_plugins.tar.bz2
+temp/
diff --git a/services/jenkins-master/README.md
b/services/jenkins-master/README.md
new file mode 100644
index 0000000..631da0b
--- /dev/null
+++ b/services/jenkins-master/README.md
@@ -0,0 +1,35 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements. See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership. The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License. You may obtain a copy of the License at -->
+
+<!--- http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied. See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+This is the Terraform setup we used to create the Jenkins master. This script
is **VERY** outdated!
+
+# Create infrastructure
+
+Warning: this will destroy the current DNS entries.
+
+```
+./init.sh
+terraform init
+terraform apply
+```
+
+
+With difference instance type (overriding variables)
+
+```
+terraform apply -var instance_type=c1.xlarge
+```
diff --git a/services/jenkins-master/deploy.sh
b/services/jenkins-master/deploy.sh
new file mode 100755
index 0000000..8ef0e04
--- /dev/null
+++ b/services/jenkins-master/deploy.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+echo "Config dir:"
+read config_dir
+
+python3 scripts/deploy_infrastructure.py --configdir "$config_dir"
--terraformdir "."
diff --git a/services/jenkins-master/download_config.sh
b/services/jenkins-master/download_config.sh
new file mode 100755
index 0000000..0d29ede
--- /dev/null
+++ b/services/jenkins-master/download_config.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+echo "Config dir:"
+read config_dir
+
+python3 scripts/jenkins_sync_config.py --jenkinsdir "$config_dir/jenkins"
--varfile "$config_dir/jenkins_config.varfile" --symlinkfile
"$config_dir/jenkins_config.symlinkfile" --secretsdir "$config_dir/secrets"
--tfvarsfile="$config_dir/infrastructure.tfvars" --mode "download"
diff --git a/services/jenkins-master/infrastructure.tf
b/services/jenkins-master/infrastructure.tf
new file mode 100644
index 0000000..3bd16ed
--- /dev/null
+++ b/services/jenkins-master/infrastructure.tf
@@ -0,0 +1,351 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+variable "key_name" {
+ type = "string"
+}
+
+variable "key_path" {
+ type = "string"
+}
+
+variable "instance_type" {
+ type = "string"
+}
+
+variable "vpc_id" {
+ type = "string"
+}
+
+variable "additional_security_group_ids" {
+ type = "list"
+}
+
+variable "jenkins_config_bucket" {
+ type = "string"
+}
+
+variable "zone_id" {
+ type = "string"
+}
+
+variable "domain" {
+ type = "string"
+}
+
+variable "shell_variables_file" {
+ type = "string"
+}
+
+# AMI IDs can be retrieved at
+# ftp://64.50.236.216/pub/ubuntu-cloud-images/query/xenial/server/released.txt
+variable "ami" {
+ type = "string"
+ default = "ami-bd8f33c5" # Ubuntu 16.04 from 20180122
+}
+
+variable "instance_name" {
+ type = "string"
+}
+
+variable "aws_availability_zone" {
+ type = "string"
+}
+
+variable "aws_region" {
+ type = "string"
+}
+
+variable "aws_access_key" {
+ type = "string"
+}
+
+variable "aws_secret_key" {
+ type = "string"
+}
+
+variable "ebs_volume_jenkins_master_state_volume_id" {
+ type = "string"
+}
+
+provider "aws" {
+ access_key = "${var.aws_access_key}"
+ secret_key = "${var.aws_secret_key}"
+ region = "${var.aws_region}"
+}
+
+# Store terraform state in S3 instead of local.
+terraform {
+ backend "s3" {
+ key = "terraform.tfstate"
+
+ # TODO: Lock statefile using dynamo db to prevent overriding
statefiles if multiple
+ # people run terraform at the same time
+ # dynamodb_table =
"terraform-state-${var.jenkins_config_bucket}-dynamo"
+
+ # Remaining config is defined in
$CONFIG_DIR/infrastructure_backend.tfvars
+ # See https://www.terraform.io/docs/backends/config.html for more
details
+ }
+}
+
+data "template_cloudinit_config" "user_data" {
+ # gzip = true
+ base64_encode = true
+
+ # Important: This part has to be in first place as it gets mapped to
+ # /var/lib/cloud/instance/scripts/part-001
+ # This is a hack, but there's no other way to reference other scripts
+ part {
+ content_type = "text/x-shellscript"
+ content = "${file("${var.shell_variables_file}")}"
+ }
+
+ # part-002
+ part {
+ content_type = "text/x-shellscript"
+ content = "${file("temp/jenkins_symlinks.sh")}"
+ }
+
+
+ # No Docker required on jenkins master
+ #part {
+ # content_type = "text/x-shellscript"
+ # content = "${file("scripts/docker_setup.sh")}"
+ #}
+ part {
+ content_type = "text/x-shellscript"
+ content = "${file("scripts/jenkins_setup.sh")}"
+ }
+}
+
+# Require in order to allow attaching policies, so called trust relationship
policy document
+resource "aws_iam_role" "jenkins_master_role" {
+ name = "jenkins_master_role"
+ assume_role_policy = <<POLICY
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Principal": {
+ "Service": "ec2.amazonaws.com"
+ },
+ "Effect": "Allow",
+ "Sid": ""
+ }
+ ]
+}
+POLICY
+}
+
+resource "aws_iam_policy" "jenkins_master_s3_read_policy" {
+ name = "jenkins_master_s3_read_policy"
+ description = "Policy to grant Jenkins Master S3 read-access to the
associated bucket"
+ policy = <<POLICY
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:GetBucketLocation",
+ "s3:ListAllMyBuckets"
+ ],
+ "Resource": "arn:aws:s3:::*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": [
+ "arn:aws:s3:::${var.jenkins_config_bucket}"
+ ]
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:GetObject"
+ ],
+ "Resource": [
+ "arn:aws:s3:::${var.jenkins_config_bucket}/*"
+ ]
+ }
+ ]
+}
+POLICY
+}
+
+resource "aws_iam_policy_attachment" "jenkins_master_s3_read_policy_attach" {
+ name = "jenkins_master_s3_read_policy_attach"
+ roles = ["${aws_iam_role.jenkins_master_role.name}"]
+ policy_arn = "${aws_iam_policy.jenkins_master_s3_read_policy.arn}"
+}
+
+resource "aws_iam_instance_profile" "jenkins_master_profile" {
+ name = "jenkins_master_profile"
+ role = "${aws_iam_role.jenkins_master_role.name}"
+}
+
+resource "aws_security_group" "allow_all_https" {
+ name = "tf_allow_all_https2"
+ description = "Allow all inbound traffic to https"
+
+ ingress {
+ from_port = 443
+ to_port = 443
+ protocol = "tcp"
+ cidr_blocks = ["0.0.0.0/0"]
+ ipv6_cidr_blocks = ["::/0"]
+ }
+}
+
+resource "aws_security_group" "allow_all_www" {
+ name = "tf_allow_all_www2"
+ description = "Allow all inbound traffic to www"
+
+ ingress {
+ from_port = 80
+ to_port = 80
+ protocol = "tcp"
+ cidr_blocks = ["0.0.0.0/0"]
+ ipv6_cidr_blocks = ["::/0"]
+ }
+}
+
+
+resource "aws_instance" "mxnet-ci" {
+ # The connection block tells our provisioner how to
+ # communicate with the resource (instance)
+ connection {
+ # The default username for our AMI
+ user = "ubuntu"
+
+ # The path to your keyfile
+ key_file = "${var.key_path}"
+ }
+
+ # subnet ID for our VPC TODO
+ # subnet_id = "${var.vpc_id}"
+ # the instance type we want, comes from rundeck
+ instance_type = "${var.instance_type}"
+
+ availability_zone = "${var.aws_availability_zone}"
+
+ # Spot instance params:
+ #spot_price = "${var.spot_price}"
+ #wait_for_fulfillment = "true"
+
+ # Lookup the correct AMI based on the region
+ # we specified
+ ami = "${var.ami}"
+
+ iam_instance_profile =
"${aws_iam_instance_profile.jenkins_master_profile.name}"
+ # The name of our SSH keypair you've created and downloaded
+ # from the AWS console.
+ #
+ # https://console.aws.amazon.com/ec2/v2/home?region=us-west-2#KeyPairs:
+ #
+ key_name = "${var.key_name}"
+
+ #vpc_security_group_ids = "${var.security_groups_ids}"
+
+ vpc_security_group_ids = ["${
+ concat(
+ list(aws_security_group.allow_all_https.id),
+ list(aws_security_group.allow_all_www.id),
+ var.additional_security_group_ids
+ )
+ }"]
+ user_data = "${data.template_cloudinit_config.user_data.rendered}"
+
+ # We set the name as a tag. Not supported for spot instances:
+ # See https://github.com/hashicorp/terraform/issues/3263
+ tags {
+ "Name" = "${var.instance_name}"
+ }
+
+
+ root_block_device {
+ volume_type = "gp2"
+ volume_size = 1000
+ #iops = 15000
+ delete_on_termination = true
+ }
+
+ # Wait for S3 bucket as it's needed during startup
+ depends_on = [
+ "aws_s3_bucket_object.jenkins_config_s3",
+ "aws_s3_bucket_object.jenkins_plugins_s3"
+ ]
+}
+
+resource "aws_s3_bucket" "jenkins_config_bucket" {
+ bucket = "${var.jenkins_config_bucket}"
+ acl = "private"
+}
+
+resource "aws_s3_bucket_object" "jenkins_config_s3" {
+ bucket = "${aws_s3_bucket.jenkins_config_bucket.id}"
+ key = "jenkins/jenkins.tar.bz2"
+ source = "temp/jenkins.tar.bz2"
+ etag = "${md5(file("temp/jenkins.tar.bz2"))}"
+}
+
+resource "aws_s3_bucket_object" "jenkins_plugins_s3" {
+ bucket = "${aws_s3_bucket.jenkins_config_bucket.id}"
+ key = "jenkins/jenkins_plugins.tar.bz2"
+ source = "temp/jenkins_plugins.tar.bz2"
+ etag = "${md5(file("temp/jenkins_plugins.tar.bz2"))}"
+}
+
+# TODO: Check for race conditions. Just in case:
+# https://github.com/hashicorp/terraform/issues/2740#issuecomment-288549352
+resource "aws_volume_attachment" "ebs_jenkins_master_state" {
+ device_name = "/dev/sdf"
+ volume_id = "${var.ebs_volume_jenkins_master_state_volume_id}"
+ instance_id = "${aws_instance.mxnet-ci.id}"
+}
+
+#output "user_data" {
+# value = "${data.template_cloudinit_config.user_data.rendered}"
+#}
+
+
+resource "aws_route53_record" "mxnet-ci" {
+ zone_id = "${var.zone_id}"
+ name = "jenkins.${var.domain}"
+ type = "A"
+ ttl = "60"
+ records = ["${aws_instance.mxnet-ci.public_ip}"]
+}
+
+resource "aws_route53_record" "mxnet-ci-private" {
+ zone_id = "${var.zone_id}"
+ name = "jenkins-priv.${var.domain}"
+ type = "A"
+ ttl = "60"
+ records = ["${aws_instance.mxnet-ci.private_ip}"]
+}
+
+output "address" {
+ value = "${aws_instance.mxnet-ci.public_dns}"
+}
+
+output "mxnet-ci" {
+ value = "${aws_route53_record.mxnet-ci.fqdn}"
+}
diff --git a/services/jenkins-master/prod/infrastructure.tfvars
b/services/jenkins-master/prod/infrastructure.tfvars
new file mode 100644
index 0000000..e34ab07
--- /dev/null
+++ b/services/jenkins-master/prod/infrastructure.tfvars
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# TODO: NOT COMPLETE!
+key_name = "REDACTED"
+key_path = "~/.ssh/REDACTED"
+instance_type = "c5.4xlarge"
+vpc_id = "REDACTED"
+
+additional_security_group_ids = [
+ "sg-d3dacfae", # VPC default
+ "sg-REDACTED" # REDACTED
+]
+
+bucket = "mxnet-ci-master"
+zone_id = "REDACTED"
+domain = "mxnet-ci.amazon-ml.com"
+instance_name = "MXNet-CI-Master"
+aws_region = "us-west-2"
diff --git a/services/jenkins-master/prod/jenkins/REDACTED-FULLY
b/services/jenkins-master/prod/jenkins/REDACTED-FULLY
new file mode 100644
index 0000000..e69de29
diff --git a/services/jenkins-master/scripts/deploy_infrastructure.py
b/services/jenkins-master/scripts/deploy_infrastructure.py
new file mode 100644
index 0000000..2bfc02e
--- /dev/null
+++ b/services/jenkins-master/scripts/deploy_infrastructure.py
@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# -*- coding: utf-8 -*-
+
+# This script automates the Jenkins master deployment process.
+
+import argparse
+import logging
+import os
+import re
+import shutil
+import subprocess
+import tarfile
+from shutil import copytree
+from tempfile import TemporaryDirectory
+
+from jenkins_config_templating import execute_config_templating,
assemble_symlink_list
+
+JENKINS_DIR_NAME = 'jenkins'
+SECRET_DIR_NAME = 'secrets'
+VARFILE_FILE_NAME = 'jenkins_config.varfile'
+SYMLINKFILE_FILE_NAME = 'jenkins_config.symlinkfile'
+
+TERRAFORM_DEPLOY_TEMP_DIR = 'temp'
+TERRAFORM_SCRIPT_NAME = 'infrastructure.tf'
+TERRAFORM_VARFILE_NAME = 'infrastructure.tfvars'
+TERRAFORM_BACKEND_VARFILE_NAME = 'infrastructure_backend.tfvars'
+
+JENKINS_PLUGINS_DIR_NAME = 'plugins'
+JENKINS_CONFIG_TAR_NAME = 'jenkins.tar.bz2'
+JENKINS_PLUGINS_TAR_NAME = 'jenkins_plugins.tar.bz2'
+JENKINS_SYMLINK_FILE_NAME = 'jenkins_symlinks.sh'
+
+STATE_TOUCH_FILE_TEMPLATE = 'touch /ebs_jenkins_state/{} \n'
+STATE_CREATE_DIR_TEMPLATE = 'mkdir -p /ebs_jenkins_state/{} \n'
+STATE_SYMLINK_TEMPLATE = 'mkdir -p /var/lib/jenkins/{} && sudo ln -s
/ebs_jenkins_state/{} /var/lib/jenkins/{} \n'
+
+
+def main():
+ logging.getLogger().setLevel(logging.DEBUG)
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-d', '--configdir',
+ help='Jenkins deployment configuration directory',
+ default='test',
+ type=str)
+
+ parser.add_argument('-tf', '--terraformdir',
+ help='Directory containing the terraform scripts',
+ default='.',
+ type=str)
+
+ args = parser.parse_args()
+
+ terraform_dir_abs = os.path.abspath(args.terraformdir)
+
+ if not os.path.isfile(os.path.join(terraform_dir_abs,
TERRAFORM_SCRIPT_NAME)):
+ raise FileNotFoundError('Unable to find terraform script. Did you
specify "--terraformdir"? {}'.
+ format(os.path.join(terraform_dir_abs,
TERRAFORM_SCRIPT_NAME)))
+
+ # Copy configuration to temp dir
+ with TemporaryDirectory() as temp_dir:
+ terraform_deploy_dir = os.path.join(terraform_dir_abs,
TERRAFORM_DEPLOY_TEMP_DIR)
+ # Create deployment temp dir
+ if not os.path.exists(terraform_deploy_dir):
+ os.makedirs(terraform_deploy_dir)
+
+ jenkins_dir = os.path.join(args.configdir, JENKINS_DIR_NAME)
+ temp_jenkins_dir = os.path.join(temp_dir, JENKINS_DIR_NAME)
+ logging.debug('Copying jenkins dir from {} to {}'.format(jenkins_dir,
temp_jenkins_dir))
+ copytree(jenkins_dir, temp_jenkins_dir, True)
+
+ # Replace placeholders with actual secrets
+ execute_config_templating(os.path.join(args.configdir,
VARFILE_FILE_NAME),
+ os.path.join(args.configdir,
SECRET_DIR_NAME), temp_jenkins_dir, 'insert',
+ update_secrets=False)
+ logging.debug('Config replaced. Result can be found at
{}'.format(temp_jenkins_dir))
+
+ # Assemble list of symlinks to be created
+ symlinks = assemble_symlink_list(os.path.join(args.configdir,
SYMLINKFILE_FILE_NAME), temp_jenkins_dir)
+
+ # Create shell script to symlink during startup
+ _create_symlink_shellscript(symlinks,
os.path.join(terraform_deploy_dir, JENKINS_SYMLINK_FILE_NAME))
+
+ # Sanity: Ensure no state is part of config
+ _validate_config_contain_no_state(symlinks, temp_jenkins_dir)
+
+ # Optional: Create backup of EBS
+ # TODO
+
+ # Compress jenkins dir to allow upload to S3
+ temp_jenkins_compressed_file = os.path.join(temp_dir,
'jenkins_config.tar.bz2')
+ temp_jenkins_compressed_plugins_file = os.path.join(temp_dir,
'jenkins_plugins.tar.bz2')
+ with tarfile.open(temp_jenkins_compressed_file, "w:bz2") as tar:
+ with tarfile.open(temp_jenkins_compressed_plugins_file, "w:bz2")
as tar_plugin:
+ for file in os.listdir(temp_jenkins_dir):
+ logging.debug('Archiving {}'.format(file))
+ # Since jenkins plugins are a few hundreds of megabytes,
store them in a second compressed archive
+ # in order to prevent uploading (~15mins) them every
single time the actual configuration is changed
+ if file != JENKINS_PLUGINS_DIR_NAME:
+ tar.add(os.path.join(temp_jenkins_dir, file),
arcname=os.path.basename(file))
+ else:
+ tar_plugin.add(os.path.join(temp_jenkins_dir, file),
arcname=os.path.basename(file))
+
+ logging.info('Copying archives to {}'.format(terraform_dir_abs))
+ # Copy generated archives to original dir
+ shutil.copy2(temp_jenkins_compressed_file,
os.path.join(terraform_deploy_dir, JENKINS_CONFIG_TAR_NAME))
+ shutil.copy2(temp_jenkins_compressed_plugins_file,
os.path.join(terraform_deploy_dir, JENKINS_PLUGINS_TAR_NAME))
+
+ # Trigger terraform
+ logging.info('Running terraform...')
+ logging.debug('Switching current work dir to {}'.format(terraform_dir_abs))
+ os.chdir(terraform_dir_abs)
+
+ # Setting up the terraform S3 backend requires to have AWS credentials in
the env vars - it's not able
+ # to access the variables file due interpolation in terraform being
enabled after initialization of
+ # the s3 backend
+ env_vars = os.environ.copy()
+ env_vars['AWS_ACCESS_KEY_ID'] =
_get_tfvars_entry(os.path.join(args.configdir, TERRAFORM_VARFILE_NAME),
+ 'aws_access_key')
+ env_vars['AWS_SECRET_ACCESS_KEY'] =
_get_tfvars_entry(os.path.join(args.configdir, TERRAFORM_VARFILE_NAME),
+ 'aws_secret_key')
+
+ p1 = subprocess.Popen('~/bin/terraform init -backend-config={}'.
+ format(os.path.join(args.configdir,
TERRAFORM_BACKEND_VARFILE_NAME)), cwd=terraform_dir_abs,
+ env=env_vars, shell=True)
+ p1.wait()
+ p2 = subprocess.Popen('~/bin/terraform apply -var-file="{}"'.
+ format(os.path.join(args.configdir,
TERRAFORM_VARFILE_NAME)), cwd=terraform_dir_abs,
+ env=env_vars, shell=True)
+ p2.wait()
+ logging.info('Deployment finished')
+
+
+def _get_tfvars_entry(tfvars_file, key):
+ # This is just a hack because I don't want to spend the time to write an
entire parser for the .tfvars format
+ with open(tfvars_file, 'r') as fp:
+ for line in fp:
+ if line.startswith(key):
+ result = re.search('"(.*)"', line).group(1)
+ return result
+
+ raise ValueError('Could not find {} in {}'.format(key, tfvars_file))
+
+
+def _create_symlink_shellscript(symlinks, target_file):
+ with open(target_file, 'w') as fp:
+ for symlink_entry in symlinks:
+ # Ensure dirs and files exist on EBS before creating symlink
+ if symlink_entry.is_dir:
+
fp.write(STATE_CREATE_DIR_TEMPLATE.format(symlink_entry.filepath))
+ else:
+
fp.write(STATE_TOUCH_FILE_TEMPLATE.format(symlink_entry.filepath))
+
+ # Create symlink
+ fp.write(STATE_SYMLINK_TEMPLATE.
+ format(os.path.dirname(symlink_entry.filepath),
symlink_entry.filepath, symlink_entry.filepath))
+
+
+def _validate_config_contain_no_state(symlinks, jenkins_config_dir):
+ for symlink_entry in symlinks:
+ path = os.path.join(jenkins_config_dir, symlink_entry.filepath)
+ if os.path.isfile(path) or os.path.isdir(path):
+ raise FileExistsError(
+ '{} is defined as state, but included in config. Remove before
continuing.'.format(path))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/services/jenkins-master/scripts/docker_setup.sh
b/services/jenkins-master/scripts/docker_setup.sh
new file mode 100755
index 0000000..87d9f38
--- /dev/null
+++ b/services/jenkins-master/scripts/docker_setup.sh
@@ -0,0 +1,163 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# To verify if these steps execute correctly, you can check
/var/log/cloud-init-output.log in the instances
+# Cloud init doesn't log anymore, so we log to user-data log and syslog
+
+# UserData script which installs docker and code deploy agent in AML / Ubuntu
+
+set -e
+set -x
+
+#Just log to cloud-init-output.log
+#exec > >(tee -a /var/log/user-data.log|logger -t user-data ) 2>&1
+
+DISTRO=$(awk -F= '/^NAME/{print $2}' /etc/os-release)
+DISTRO=${DISTRO//\"/}
+
+echo "Running on $DISTRO"
+
+function install_docker_ubuntu() {
+ #apt-get -y install docker docker.io
+ export DEBIAN_FRONTEND=noninteractive
+ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+ add-apt-repository \
+ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+ $(lsb_release -cs) \
+ stable"
+ apt-get update
+ apt-get -y install docker-ce
+ service docker restart
+ usermod -a -G docker ubuntu
+}
+
+function install_docker_aml() {
+ yum -y install docker.x86_64
+ sudo usermod -a -G docker ec2-user
+ service docker restart
+}
+
+function install_code_deploy_agent() {
+ # Install code deploy agent
+ pushd .
+ TMP_INSTALL=/tmp/code_deploy
+ mkdir -p $TMP_INSTALL
+ cd $TMP_INSTALL
+ wget https://aws-codedeploy-eu-central-1.s3.amazonaws.com/latest/install
+ chmod +x ./install
+ ./install auto
+ popd
+ rm -rf $TMP_INSTALL
+}
+
+
+
+case $DISTRO in
+"Ubuntu")
+ export DEBIAN_FRONTEND=noninteractive
+ apt-get update
+ apt-get -y install ruby wget vim-nox silversearcher-ag htop
+ install_code_deploy_agent
+ install_docker_ubuntu
+ ;;
+"Amazon Linux AMI")
+ echo "AL"
+ install_docker_aml
+ install_code_deploy_agent
+ yum -y install git python35
+ ;;
+*)
+ echo "?"
+ ;;
+esac
+
+
+
+
+
+# Use the ephemeral storage for docker so we don't run out of space
+function setup_docker_ephemeral() {
+ if mountpoint -q -- /mnt; then
+ echo "Using /mnt/docker for docker storage"
+ service docker stop
+ mkdir -p /mnt/docker
+ mount -o bind /mnt/docker/ /var/lib/docker/
+ service docker start
+ fi
+}
+
+
+function setup_ephemeral_raid() {
+ METADATA_URL_BASE="http://169.254.169.254/2016-09-02/";
+ DRIVE_SCHEME=`mount | perl -ne 'if(m#/dev/(xvd|sd).\d?#) { print "$1\n";
exit}'`
+
+ drives=""
+ ephemeral_count=0
+ ephemerals=$(curl --silent
$METADATA_URL_BASE/meta-data/block-device-mapping/ | grep ephemeral)
+ for e in $ephemerals; do
+ echo "Probing $e .."
+ device_name=$(curl --silent
$METADATA_URL_BASE/meta-data/block-device-mapping/$e)
+ # might have to convert 'sdb' -> 'xvdb'
+ device_name=$(echo $device_name | sed "s/sd/$DRIVE_SCHEME/")
+ device_path="/dev/$device_name"
+
+ # test that the device actually exists since you can request more
ephemeral drives than are available
+ # for an instance type and the meta-data API will happily tell you it
exists when it really does not.
+ if [ -b $device_path ]; then
+ echo "Detected ephemeral disk: $device_path"
+ drives="$drives $device_path"
+ ephemeral_count=$((ephemeral_count + 1 ))
+ umount $device_path || true
+ else
+ echo "Ephemeral disk $e, $device_path is not present. skipping"
+ fi
+ done
+
+ if [ "$ephemeral_count" = 0 ]; then
+ echo "No ephemeral disk detected."
+ return 1
+ fi
+
+ # overwrite first few blocks in case there is a filesystem, otherwise
mdadm will prompt for input
+ for drive in $drives; do
+ dd if=/dev/zero of=$drive bs=4096 count=1024
+ done
+
+ partprobe || true
+ # Force in case there's only one drive
+ mdadm --create --force --verbose /dev/md0 --level=0 -c256
--raid-devices=$ephemeral_count $drives
+ echo DEVICE $drives | tee /etc/mdadm.conf
+ mdadm --detail --scan | tee -a /etc/mdadm.conf
+ blockdev --setra 65536 /dev/md0
+ mkfs -t ext4 -m 0 /dev/md0
+ mount -t ext4 -o noatime /dev/md0 /mnt
+
+ # Remove xvdb/sdb from fstab
+ chmod 777 /etc/fstab
+ sed -i "/${DRIVE_SCHEME}b/d" /etc/fstab
+
+ # Make raid appear on reboot
+ echo "/dev/md0 /mnt ext4 noatime 0 0" | tee -a /etc/fstab
+ return 0
+}
+
+# Add your custom initialization code below
+(setup_ephemeral_raid && setup_docker_ephemeral) || true
+
+echo "UserData initialization is done"
diff --git a/services/jenkins-master/scripts/jenkins_config_templating.py
b/services/jenkins-master/scripts/jenkins_config_templating.py
new file mode 100644
index 0000000..abf8c79
--- /dev/null
+++ b/services/jenkins-master/scripts/jenkins_config_templating.py
@@ -0,0 +1,275 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# -*- coding: utf-8 -*-
+
+# This script serves the purpose to replace sensitive parts in the jenkins
configuration with placeholders in order
+# to allow the configuration to be published to a public repository
+
+import argparse
+import filecmp
+import glob
+import json
+import logging
+import os
+import pathlib
+import shutil
+from collections import namedtuple
+
+from lxml import etree
+
+SECRET_ENTRY_KEYS = ['filepath', 'xpath', 'secret', 'placeholder']
+SecretEntry = namedtuple('SecretEntry', SECRET_ENTRY_KEYS)
+
+SYMLINK_ENTRY_KEYS = ['filepath', 'is_dir']
+SymlinkEntry = namedtuple('SymlinkEntry', SYMLINK_ENTRY_KEYS)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-vf', '--varfile',
+ help='Location of the variable file',
+ type=str)
+
+ parser.add_argument('-sf', '--symlinkfile',
+ help='Location of the symlink file',
+ type=str)
+
+ parser.add_argument('-sd', '--secretsdir',
+ help='Location of the directory containing secrets',
+ type=str)
+
+ parser.add_argument('-jd', '--jenkinsdir',
+ help='Location of the jenkins directory',
+ type=str)
+
+ parser.add_argument('-m', '--mode',
+ help='"remove" or "insert" credentials',
+ default='insert',
+ type=str)
+
+ args = parser.parse_args()
+
+ execute_config_templating(args.varfile, args.secretsdir, args.jenkinsdir,
args.mode, update_secrets=False)
+ # TODO: Add symlink list creation
+
+
+def execute_config_templating(varfile, secretsdir, jenkinsdir, mode,
update_secrets):
+ """
+ Execute config templating that inserts or removes secrets from a jenkins
configuration directory
+ :param varfile: File containing the actual variables that should be used
during replacement
+ :param secretsdir: Directory containing all files that should be just
copied or removed as secrets
+ :param jenkinsdir: Jenkins configuration directory
+ :param mode: 'insert' replaces placeholders with actual values. 'remove'
removes these values and
+ inserts placeholders
+ :param update_secrets: Update secrets if mode == 'remove' and actual
secrets differ from the stored ones
+ :return:
+ """
+ secret_entries = read_secret_entires(varfile)
+ logging.debug('Found {} secret entries to be
replaced'.format(len(secret_entries)))
+
+ # Prepare by finding all unique identifiers. This is required because XML
parsers do not allow in-place
+ # replacements. Instead, we're reading a unique identifier under the
specified xpath and then try to
+ # replace it in-place by using search-and-replace. This method will be
aborted if the current value
+ # has been found multiple times within the same file.
+ for root, dirs, files in os.walk(secretsdir, topdown=False):
+ for name in files:
+ original_path = os.path.join(root, name)
+ rel_path = os.path.relpath(original_path, secretsdir)
+ temp_path = os.path.join(jenkinsdir, rel_path)
+
+ if mode == 'insert':
+ pathlib.Path(os.path.dirname(temp_path)).mkdir(parents=True,
exist_ok=True)
+ shutil.copyfile(original_path, temp_path)
+ elif mode == 'remove':
+ # Check if secret does not exist anymore
+ if not os.path.isfile(temp_path):
+ if update_secrets:
+ logging.info('Deleting secret {} because it has been
removed on target'.format(rel_path))
+ os.remove(original_path)
+ else:
+ raise ValueError('Secret {} has been
deleted'.format(rel_path))
+
+ # Check if secrets are the same or have to be updated
+ if not filecmp.cmp(temp_path, original_path):
+ if update_secrets:
+ logging.info('Replacing secret {} due to changed
content'.format(rel_path))
+ shutil.copyfile(temp_path, original_path)
+ else:
+ raise ValueError('Secret {} contains changed
content'.format(rel_path))
+
+ os.remove(temp_path)
+ else:
+ raise ValueError('Mode {} unknown'.format(mode)) # TODO check
this previously
+
+ # Check if any files in the secrets-dir are left that didn't exist in the
previous config. Unfortunately,
+ # we can't verify if secrets outside the secrets-dir have been added.
+ if mode == 'remove':
+ temp_secrets_dir = os.path.join(jenkinsdir, 'secrets')
+ for root, dirs, files in os.walk(temp_secrets_dir, topdown=False):
+ for name in files:
+ temp_path = os.path.join(root, name)
+ rel_path = os.path.relpath(temp_path, jenkinsdir)
+ original_path = os.path.join(secretsdir, rel_path)
+
+ if update_secrets:
+ logging.info('Adding new secret at {}'.format(rel_path))
+ shutil.copyfile(temp_path, original_path)
+ else:
+ raise ValueError('New secret at {}'.format(rel_path))
+ shutil.rmtree(temp_secrets_dir)
+
+ for secret_entry in secret_entries:
+ temp_file_path = os.path.join(jenkinsdir, secret_entry.filepath)
+ if os.path.isfile(temp_file_path):
+ element = etree.parse(temp_file_path).xpath(secret_entry.xpath)
+
+ # Check if xpath delivers multiple results. The xpath should only
match once
+ if len(element) == 1:
+ current_value = element[0].text
+ if not current_value.strip():
+ raise ValueError('Element {} at {}:{} is not a text field'.
+ format(current_value, temp_file_path,
secret_entry.xpath))
+
+ if mode == 'insert':
+ expected_value = secret_entry.placeholder
+ target_value = secret_entry.secret
+ elif mode == 'remove':
+ expected_value = secret_entry.secret
+ target_value = secret_entry.placeholder
+ else:
+ raise ValueError('Mode {} unknown'.format(mode))
+
+ if current_value == expected_value:
+ logging.debug(
+ 'Replacing {} with {} at {}:{}'.format(current_value,
target_value, secret_entry.xpath,
+ temp_file_path))
+ _replace_values(current_value, target_value,
temp_file_path)
+
+ elif current_value == target_value:
+ logging.info('Target value {} already present. Skipping
{}:{}'.
+ format(target_value, secret_entry.xpath,
temp_file_path))
+ continue
+ else:
+ raise ValueError('Current value "{}" does not match
expected value "{}" in {}:{}'.format(
+ current_value, expected_value, secret_entry.xpath))
+ elif len(element) == 0:
+ raise ValueError('Element at {}:{} not
found'.format(temp_file_path, secret_entry.xpath))
+ else:
+ raise ValueError('1 Element expected, {} found at {}:{}'.
+ format(len(element), temp_file_path,
secret_entry.xpath))
+ else:
+ raise FileNotFoundError('Could not find file
{}'.format(temp_file_path))
+
+
+def assemble_symlink_list(symlink_file, jenkinsdir):
+ """
+ Assemble a list of files that should be symlinked during startup,
providing support for state files on EBS
+ :param symlink_file: File containing path expressions to describe the
symlinked files and dirs
+ :param jenkinsdir: Jenkins configuration directory
+ :return: Array of SymlinkEntry
+ """
+ symlink_config = read_symlink_entries(symlink_file)
+ logging.debug('Found {} symlink entries'.format(len(symlink_config)))
+
+ symlinks = []
+
+ for symlink_entry in symlink_config:
+ input_path_expression = os.path.join(jenkinsdir,
symlink_entry.filepath)
+ result_paths = []
+
+ if '*' in symlink_entry.filepath:
+ # If path contains a wildcard, search for all results
+ wildcard_path_split = input_path_expression.split('*')
+ if len(wildcard_path_split) == 1:
+ result_paths = glob.glob(input_path_expression)
+ elif len(wildcard_path_split) == 2:
+ # If wildcard is in the middle and target directories don't
exist, result would be empty.
+ # Instead, iterate manually
+ for partial_path in
glob.glob(os.path.join(wildcard_path_split[0], '*')):
+ result_paths.append(os.path.join(partial_path,
wildcard_path_split[1].lstrip('/')))
+ else:
+ raise ValueError('Symlink expression may only contain one
wildcard')
+ logging.debug('Resolving {} to {}'.format(symlink_entry.filepath,
result_paths))
+ else:
+ result_paths = [input_path_expression]
+
+ for abs_path in result_paths:
+ rel_path = os.path.relpath(abs_path, jenkinsdir)
+ symlinks.append(SymlinkEntry(rel_path, symlink_entry.is_dir))
+
+ return symlinks
+
+
+def read_secret_entires(varfile):
+ """
+ Read SecretEntry from varfile
+ :param varfile: File containing SecretEntries as JSON
+ :return: Array of SecretEntry
+ """
+ with open(varfile, 'r') as fp:
+ secret_dict_raw = json.load(fp)
+ secrets = []
+ for secret_entry_dict in secret_dict_raw:
+ # Import values in the same order as expected in SecretEntry.
Reason being that the values are expected
+ # to be inserted in the same order as defined previously.
+ secrets.append(SecretEntry(*[secret_entry_dict[k] for k in
SECRET_ENTRY_KEYS]))
+ return secrets
+
+
+def read_symlink_entries(varfile):
+ """
+ Read SymlinkEntry from varfile
+ :param varfile: File containing SymlinkEntries as JSON
+ :return: Array of SymlinkEntries
+ """
+ with open(varfile, 'r') as fp:
+ symlink_dict_raw = json.load(fp)
+ symlinks = []
+ for symlink_entry_dict in symlink_dict_raw:
+ # Import values in the same order as expected in SecretEntry.
Reason being that the values are expected
+ # to be inserted in the same order as defined previously.
+ symlinks.append(SymlinkEntry(*[symlink_entry_dict[k] for k in
SYMLINK_ENTRY_KEYS]))
+ return symlinks
+
+
+def _replace_values(current_value, target_value, temp_file_path):
+ # Only execute changes on the temp file
+ with open(temp_file_path, 'r+') as temp_fh:
+ temp_file_content = temp_fh.read()
+
+ # Check if current value is unique within the temp file to prevent
injection attacks
+ nb_occurrences_temp = temp_file_content.count(current_value)
+ if nb_occurrences_temp != 1:
+ raise ValueError(
+ '1 occurrence of current value {} in file {} expected, {}
found'.format(
+ current_value,
+ temp_file_path,
+ nb_occurrences_temp))
+
+ temp_file_content_replaced = temp_file_content.replace(current_value,
target_value)
+
+ # Write replaced content back to the temp file
+ temp_fh.seek(0)
+ temp_fh.write(temp_file_content_replaced)
+ temp_fh.truncate()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/services/jenkins-master/scripts/jenkins_setup.sh
b/services/jenkins-master/scripts/jenkins_setup.sh
new file mode 100755
index 0000000..393ff29
--- /dev/null
+++ b/services/jenkins-master/scripts/jenkins_setup.sh
@@ -0,0 +1,120 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -x
+set -e
+
+# Just log to cloud-init-output.log
+# exec > >(tee -a /var/log/user-data.log|logger -t user-data ) 2>&1
+
+export DEBIAN_FRONTEND=noninteractive
+
+#########################
+# Import variables
+source /var/lib/cloud/instance/scripts/part-001
+
+# Validate that environment variables are set
+if [ -z "$DNS_ADDRESS" ]; then
+ echo "Need to set DNS_ADDRESS"
+ exit 1
+fi
+
+if [ -z "$S3_MASTER_BUCKET" ]; then
+ echo "Need to set S3_MASTER_BUCKET"
+ exit 1
+fi
+
+#########################
+
+#########################
+# Hostname
+echo $DNS_ADDRESS > /etc/hostname
+hostname $DNS_ADDRESS
+echo 'search $DNS_ADDRESS' >> /etc/resolv.conf
+
+#########################
+# Install jenkins. Use weekly instead of stable due to high number of security
vulnerabilities
+wget -q -O - https://pkg.jenkins.io/debian/jenkins.io.key | sudo apt-key add -
+echo deb http://pkg.jenkins.io/debian binary/ >
/etc/apt/sources.list.d/jenkins.list
+apt-get update
+apt-get -y install openjdk-8-jre-headless
+apt-get -y install jenkins mailutils
+service jenkins stop
+
+#########################
+rm -rf /var/lib/jenkins
+mkdir /var/lib/jenkins
+chown -R jenkins.jenkins /var/lib/jenkins
+#########################
+
+#########################
+#Port 1-1000 are only acccessible as root. Jenkins does not run as root, thus
port forwarding is
+#required
+network_interface=$(ip link | awk -F: '$0 !~ "lo|vir|wl|^[^0-9]"{print $2}')
+iptables -t nat -I PREROUTING -p tcp -i $network_interface --dport 80 -j
REDIRECT --to-ports 8080
+iptables -t nat -I PREROUTING -p tcp -i $network_interface --dport 443 -j
REDIRECT --to-ports 8081
+apt-get -y install iptables-persistent
+#########################
+
+#########################
+# Attach Jenkins state EBS volume
+mkdir /ebs_jenkins_state
+
+# Wait until volume mounted
+until [ -e /dev/nvme1n1 ]
+do
+ echo 'Waiting for /dev/nvme1n1 to be mounted'
+ sleep 1
+done
+
+mount /dev/nvme1n1 /ebs_jenkins_state/
+
+# Ensure volume is mounted upon restart
+sudo su root -c '(crontab -l 2>/dev/null; echo "@reboot /bin/mount
/dev/nvme1n1 /ebs_jenkins_state") | crontab -'
+
+chown -R jenkins.jenkins /ebs_jenkins_state
+#########################
+
+#########################
+# Unpack preconfigured jenkins
+apt-get -y install awscli
+tmpdir=$(mktemp -d)
+aws s3 cp --quiet s3://$S3_MASTER_BUCKET/jenkins/jenkins.tar.bz2 $tmpdir
+aws s3 cp --quiet s3://$S3_MASTER_BUCKET/jenkins/jenkins_plugins.tar.bz2
$tmpdir
+
+# Softlink state and cache files
+source /var/lib/cloud/instance/scripts/part-002
+chown -R jenkins.jenkins /ebs_jenkins_state
+
+# Copy preconfigured jenkins
+tar -C /var/lib/jenkins -xjf $tmpdir/jenkins.tar.bz2
+tar -C /var/lib/jenkins -xjf $tmpdir/jenkins_plugins.tar.bz2
+chown -R jenkins.jenkins /var/lib/jenkins
+#########################
+
+
+
+#########################
+# Set jenkins arguments (enable https)
+sed -i 's#JENKINS_ARGS.*#JENKINS_ARGS="--webroot=/var/cache/$NAME/war
--httpPort=8080 --httpsPort=8081"#' /etc/default/jenkins
+sed -i 's#JAVA_ARGS.*#JAVA_ARGS="-Xmx8192m -Xms512m
-XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC
-Djava.awt.headless=true"#' /etc/default/jenkins
+#########################
+
+service jenkins start
+echo "Jenkins setup completed successfully"
diff --git a/services/jenkins-master/scripts/jenkins_sync_config.py
b/services/jenkins-master/scripts/jenkins_sync_config.py
new file mode 100644
index 0000000..4bd3740
--- /dev/null
+++ b/services/jenkins-master/scripts/jenkins_sync_config.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# -*- coding: utf-8 -*-
+
+# This script synchronizes the local config with the running configuration on
a jenkins master server
+
+import argparse
+import glob
+import os
+import re
+import shutil
+from distutils.dir_util import copy_tree
+from tempfile import TemporaryDirectory
+
+from jenkins_config_templating import execute_config_templating,
read_symlink_entries
+
+BASH_SCRIPT_JENKINS_TO_TEMP = \
+ 'ssh-keygen -R {}; ssh -C ubuntu@{} "bash -s" <<EOS \n' \
+ 'sudo rm -rf /home/ubuntu/jenkins; mkdir -p /home/ubuntu/jenkins; \n' \
+ 'sudo cp -RP --verbose /var/lib/jenkins/* /home/ubuntu/jenkins \n' \
+ 'sudo chown -R ubuntu.ubuntu /home/ubuntu/jenkins; \n' \
+ 'find /home/ubuntu/jenkins/ -type l -delete; \n' \
+ 'EOS'
+
+BASH_SCRIPT_DOWNLOAD_TEMP = 'rsync --delete -zvaP ubuntu@{}:jenkins/ {}'
+BASH_SCRIPT_SYNC_LOCAL = 'rsync --delete -zvaP {}/* {}'
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-jd', '--jenkinsdir',
+ help='Location of the jenkins directory',
+ type=str)
+
+ parser.add_argument('-vf', '--varfile',
+ help='Location of the variable file',
+ type=str)
+
+ parser.add_argument('-sf', '--symlinkfile',
+ help='Location of the symlink file',
+ type=str)
+
+ parser.add_argument('-sd', '--secretsdir',
+ help='Location of the directory containing secrets',
+ type=str)
+
+ parser.add_argument('-tf', '--tfvarsfile',
+ help='Location of the terraform variable file',
+ type=str)
+
+ parser.add_argument('-m', '--mode',
+ help='"download" or "upload" config',
+ type=str)
+
+ args = parser.parse_args()
+
+ jenkins_sync_config(args.mode, args.jenkinsdir, args.varfile,
args.symlinkfile, args.secretsdir, args.tfvarsfile)
+
+
+def jenkins_sync_config(mode, jenkins_dir, var_file, symlink_file,
secrets_dir, tfvars_file):
+ # secret_entries = read_secret_entires(var_file) #TODO: Verify no new
secrets have been downloaded
+ symlink_config = read_symlink_entries(symlink_file)
+ jenkins_address = 'jenkins.' + _get_tfvars_entry(tfvars_file, 'domain')
+
+ with TemporaryDirectory() as temp_dir:
+ if mode == 'download':
+ # Copy config to temp dir on jenkins master to avoid permission
issues due to owner being jenkins while
+ # rsync is logging in as ubuntu. We're using bash scripts instead
of an SSH client because all python
+ # libraries for SSH usage are having trouble when we supply a
custom rsa key instead ouf using id_rsa
+ # TODO: Use proper SSH client
+ bash_jenkins_to_temp_cmd =
BASH_SCRIPT_JENKINS_TO_TEMP.format(jenkins_address, jenkins_address)
+ os.system(bash_jenkins_to_temp_cmd)
+
+ # Copy old jenkins to local temp dir to allow rsync and thus speed
up the download process due to diff
+ copy_tree(jenkins_dir, temp_dir)
+ bash_jenkins_download_cmd =
BASH_SCRIPT_DOWNLOAD_TEMP.format(jenkins_address, temp_dir)
+ os.system(bash_jenkins_download_cmd)
+
+ # Delete state files. Symlinks are already deleted before config
is downloaded, but there might be new dirs
+ # which were not symlinked yet.
+ _delete_state_files(symlink_config, temp_dir)
+
+ # Remove secrets according to secret config
+ execute_config_templating(var_file, secrets_dir, temp_dir,
'remove', update_secrets=True)
+
+ # TODO Optional: Verify no new secrets have been downloaded
+
+ # Move new config to configdir
+ bash_sync_local_cmd = BASH_SCRIPT_SYNC_LOCAL.format(temp_dir,
jenkins_dir)
+ os.system(bash_sync_local_cmd)
+ else:
+ raise ValueError('Mode {} not supported'.format(mode))
+
+
+def _delete_state_files(symlink_config, jenkins_dir):
+ for symlink_entry in symlink_config:
+ result_paths = glob.glob(os.path.join(jenkins_dir,
symlink_entry.filepath))
+ for path in result_paths:
+ if symlink_entry.is_dir:
+ shutil.rmtree(path)
+ else:
+ os.remove(path)
+
+
+def _get_tfvars_entry(tfvars_file, key):
+ # This is just a hack because I don't want to spend the time to write an
entire parser for the .tfvars format
+ with open(tfvars_file, 'r') as fp:
+ for line in fp:
+ if line.startswith(key):
+ result = re.search('"(.*)"', line).group(1)
+ return result
+
+ raise ValueError('Could not find {} in {}'.format(key, tfvars_file))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/services/jenkins-master/scripts/sync_ci_to_host.sh
b/services/jenkins-master/scripts/sync_ci_to_host.sh
new file mode 100755
index 0000000..3e290c5
--- /dev/null
+++ b/services/jenkins-master/scripts/sync_ci_to_host.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#
+# Sync modifications to jenkins configuration to local
+#
+
+
+set -e
+set -x
+
+# Work around permissions of /var/lib/jenkins
+ssh -C ubu...@jenkins.mxnet-ci.amazon-ml.com "bash -s" <<EOS
+set -e
+set -x
+sudo su
+mkdir -p /home/ubuntu/jenkins
+rsync --delete --exclude="workspace/" --exclude="updates/" --exclude="logs/"
--exclude=".cache/" --exclude="fingerprints/"
--exclude="org.jenkinsci.plugins.github.GitHubPlugin.cache/"
--exclude="builds/" -vaP /var/lib/jenkins/ /home/ubuntu/jenkins
+chown -R ubuntu.ubuntu /home/ubuntu/jenkins
+EOS
+
+rsync --delete -zvaP ubu...@jenkins.mxnet-ci.amazon-ml.com:jenkins/ jenkins/
+rm -rf jenkins/.cache
+rm -rf jenkins/logs
+rm -rf jenkins/fingerprints
+rm -rf jenkins/org.jenkinsci.plugins.github.GitHubPlugin.cache/*
+
+
+
+ssh -C ubu...@jenkins.mxnet-ci.amazon-ml.com "bash -s" <<EOS
+rm -rf /home/ubuntu/jenkins
+EOS
diff --git a/services/jenkins-master/scripts/sync_host_to_ci.sh
b/services/jenkins-master/scripts/sync_host_to_ci.sh
new file mode 100755
index 0000000..de7a354
--- /dev/null
+++ b/services/jenkins-master/scripts/sync_host_to_ci.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#
+# Sync modifications to jenkins configuration to local
+#
+
+set -e
+set -x
+# Create temporary directory on the CI
+ssh -C ubu...@jenkins.mxnet-ci.amazon-ml.com "bash -s" << EOS
+set -e
+set -x
+sudo su
+rm -rf /home/ubuntu/jenkins/
+mkdir /home/ubuntu/jenkins/
+chown ubuntu.ubuntu /home/ubuntu/jenkins/
+EOS
+
+# Copy host files to the temp CI dir
+rsync -zvaP jenkins/ ubu...@jenkins.mxnet-ci.amazon-ml.com:jenkins/
+
+# Stop running jenkins, preserve state-informations, deploy the changes and
start Jenkins
+ssh -C ubu...@jenkins.mxnet-ci.amazon-ml.com "bash -s" <<EOF
+set -e
+set -x
+sudo su
+service jenkins stop
+mv -t /home/ubuntu/jenkins /var/lib/jenkins/workspace/
/var/lib/jenkins/updates/
+/var/lib/jenkins/logs/
+/var/lib/jenkins/.cache//var/lib/jenkins/fingerprints//var/lib/jenkins/org.jenkinsci.plugins.github.GitHubPlugin.cache/
+/var/lib/jenkins/builds/
+rsync -vaP /home/ubuntu/jenkins/ /var/lib/jenkins/
+rm -rf /home/ubuntu/jenkins/
+chown -R jenkins.jenkins /var/lib/jenkins/
+service jenkins start
+EOF
+
diff --git a/services/jenkins-master/test/infrastructure.tfvars
b/services/jenkins-master/test/infrastructure.tfvars
new file mode 100644
index 0000000..989ef33
--- /dev/null
+++ b/services/jenkins-master/test/infrastructure.tfvars
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+aws_access_key = "REDACTED"
+aws_secret_key = "REDACTED"
+key_name = "REDACTED"
+key_path = "~/.ssh/REDACTED"
+instance_type = "c5.4xlarge"
+vpc_id = "02c00d7b"
+
+additional_security_group_ids = [
+ "sg-5d83d421", # VPC default
+ "sg-REDACTED" # REDACTED
+]
+
+shell_variables_file = "test/variables.sh"
+jenkins_config_bucket = "mxnet-ci-master-dev"
+zone_id = "REDACTED"
+domain = "mxnet-ci-dev.amazon-ml.com"
+instance_name = "MXNet-CI-Master"
+aws_region = "us-west-2"
+
+# EBS volume and AZ have to be the same
+aws_availability_zone = "us-west-2b"
+ebs_volume_jenkins_master_state_volume_id = "vol-REDACTED"
diff --git a/services/jenkins-master/test/infrastructure_backend.tfvars
b/services/jenkins-master/test/infrastructure_backend.tfvars
new file mode 100644
index 0000000..8ad16f3
--- /dev/null
+++ b/services/jenkins-master/test/infrastructure_backend.tfvars
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Terraform backend configuration. See See
https://www.terraform.io/docs/backends/config.html for
+# more details. Reason being that config can not be interpolated during
backend init
+
+bucket = "mxnet-ci-master-dev-tfstate"
+region = "us-west-2"
diff --git a/services/jenkins-master/test/jenkins/REDACYED-FULLY
b/services/jenkins-master/test/jenkins/REDACYED-FULLY
new file mode 100644
index 0000000..e69de29
diff --git a/services/jenkins-master/test/jenkins_config.symlinkfile
b/services/jenkins-master/test/jenkins_config.symlinkfile
new file mode 100644
index 0000000..80144ac
--- /dev/null
+++ b/services/jenkins-master/test/jenkins_config.symlinkfile
@@ -0,0 +1,55 @@
+[
+ {
+ "filepath": "caches",
+ "is_dir": 1
+ },
+ {
+ "filepath": "fingerprints",
+ "is_dir": 1
+ },
+ {
+ "filepath": "jobs/*/branches",
+ "is_dir": 1
+ },
+ {
+ "filepath": "jobs/*/builds",
+ "is_dir": 1
+ },
+ {
+ "filepath": "jobs/*/indexing",
+ "is_dir": 1
+ },
+ {
+ "filepath": "logs",
+ "is_dir": 1
+ },
+ {
+ "filepath": "users",
+ "is_dir": 1
+ },
+ {
+ "filepath": "workspace",
+ "is_dir": 1
+ },
+ {
+ "filepath": "updates",
+ "is_dir": 1
+ },
+
+ {
+ "filepath":
"org.jenkinsci.plugins.workflow.flow.FlowExecutionList.xml",
+ "is_dir": 0
+ },
+ {
+ "filepath": "queue.xml.bak",
+ "is_dir": 0
+ },
+ {
+ "filepath": "queue.xml",
+ "is_dir": 0
+ },
+ {
+ "filepath": "jobs/*/nextBuildNumber",
+ "is_dir": 0
+ }
+]
diff --git a/services/jenkins-master/test/jenkins_config.varfile
b/services/jenkins-master/test/jenkins_config.varfile
new file mode 100644
index 0000000..c0c5885
--- /dev/null
+++ b/services/jenkins-master/test/jenkins_config.varfile
@@ -0,0 +1,8 @@
+[
+ {
+ "filepath": "config.xml",
+ "placeholder": "[GitHubSecurityRealm/clientSecret]",
+ "secret": "REDACTED",
+ "xpath":
"/hudson/securityRealm[@class='org.jenkinsci.plugins.GithubSecurityRealm']/clientSecret"
+ }
+]
diff --git a/services/jenkins-master/test/secrets/REDACTED-FULLY
b/services/jenkins-master/test/secrets/REDACTED-FULLY
new file mode 100644
index 0000000..e69de29
diff --git a/services/jenkins-master/test/variables.sh
b/services/jenkins-master/test/variables.sh
new file mode 100644
index 0000000..5fcc90c
--- /dev/null
+++ b/services/jenkins-master/test/variables.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+export DNS_ADDRESS="mxnet-ci-dev.amazon-ml.com"
+export S3_MASTER_BUCKET="mxnet-ci-master-dev"
