This is an automated email from the ASF dual-hosted git repository. dmysakovets pushed a commit to branch DLAB-668 in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
commit e31002d1d3b9208a17be85deab706fbdc5c15e0e Author: Dyoma33 <[email protected]> AuthorDate: Tue Jul 2 11:19:07 2019 +0300 [DLAB-668] Added terraform scripts for notebooks and computational resources --- .../terraform/gcp/main/main.tf | 88 ++++++++++++++++++ .../terraform/gcp/main/variables.tf | 55 +++++++++++ .../terraform/gcp/modules/common/iam.tf | 36 ++++++++ .../terraform/gcp/modules/common/network.tf | 39 ++++++++ .../terraform/gcp/modules/common/variables.tf | 39 ++++++++ .../terraform/gcp/modules/data_engine/instance.tf | 101 +++++++++++++++++++++ .../terraform/gcp/modules/data_engine/variables.tf | 35 +++++++ .../terraform/gcp/modules/dataproc/instance.tf | 42 +++++++++ .../terraform/gcp/modules/dataproc/variables.tf | 39 ++++++++ .../terraform/gcp/modules/notebook/instance.tf | 65 +++++++++++++ .../terraform/gcp/modules/notebook/variables.tf | 29 ++++++ 11 files changed, 568 insertions(+) diff --git a/infrastructure-provisioning/terraform/gcp/main/main.tf b/infrastructure-provisioning/terraform/gcp/main/main.tf new file mode 100644 index 0000000..3cbca78 --- /dev/null +++ b/infrastructure-provisioning/terraform/gcp/main/main.tf @@ -0,0 +1,88 @@ +provider "google" { + credentials = "${var.credentials}" + project = "${var.project_name}" + region = "${var.region_var}" + zone = "${var.zone_var}" +} + +module "common" { + source = "../modules/common" + project_tag = "${var.project_tag}" + endpoint_tag = "${var.endpoint_tag}" + user_tag = "${var.user_tag}" + custom_tag = "${var.custom_tag}" + product = "${var.product_name}" + region = "${var.region_var}" + vpc_name = "${var.vpc_name}" + fw_ingress = "${var.fw_ingress}" + fw_egress_public = "${var.fw_egress_public}" + fw_egress_private = "${var.fw_egress_private}" + network_tag = "${var.network_tag}" + cidr_range = "${var.cidr_range}" + traefik_cidr = "${var.traefik_cidr}" +} + +module "notebook" { + source = "../modules/notebook" + project_tag = "${var.project_tag}" + endpoint_tag = "${var.endpoint_tag}" + user_tag = "${var.user_tag}" + custom_tag = "${var.custom_tag}" + product = "${var.product_name}" + notebook_name = "${var.notebook_name}" + zone_var = "${var.zone_var}" + vpc_name = "${var.vpc_name}" + subnet_name = "${var.subnet_name}" + network_tag = "${var.network_tag}" + sa_email = "${var.sa_email}" + ami = "${var.ami}" + machine_type = "${var.machine_type}" + ssh_key = "${var.ssh_key}" + gpu_accelerator = "${var.gpu_accelerator}" +} + +module "data_engine" { + source = "../modules/data_engine" + project_tag = "${var.project_tag}" + endpoint_tag = "${var.endpoint_tag}" + user_tag = "${var.user_tag}" + custom_tag = "${var.custom_tag}" + product = "${var.product_name}" + notebook_name = "${var.notebook_name}" + zone_var = "${var.zone_var}" + vpc_name = "${var.vpc_name}" + subnet_name = "${var.subnet_name}" + network_tag = "${var.network_tag}" + sa_email = "${var.sa_email}" + ami = "${var.ami}" + ssh_key = "${var.ssh_key}" + gpu_accelerator = "${var.gpu_accelerator}" + cluster_name = "${var.cluster_name}" + total_count = "${var.total_count}" + master_shape = "${var.master_shape}" + slave_shape = "${var.slave_shape}" +} + +module "dataproc" { + source = "../modules/dataproc" + region = "${var.region_var}" + project_tag = "${var.project_tag}" + endpoint_tag = "${var.endpoint_tag}" + user_tag = "${var.user_tag}" + custom_tag = "${var.custom_tag}" + product = "${var.product_name}" + notebook_name = "${var.notebook_name}" + zone_var = "${var.zone_var}" + vpc_name = "${var.vpc_name}" + subnet_name = "${var.subnet_name}" + network_tag = "${var.network_tag}" + sa_email = "${var.sa_email}" + ami = "${var.ami}" + ssh_key = "${var.ssh_key}" + gpu_accelerator = "${var.gpu_accelerator}" + cluster_name = "${var.cluster_name}" + total_count = "${var.total_count}" + master_shape = "${var.master_shape}" + slave_shape = "${var.slave_shape}" + preemptible_count = "${var.preemptible_count}" +} \ No newline at end of file diff --git a/infrastructure-provisioning/terraform/gcp/main/variables.tf b/infrastructure-provisioning/terraform/gcp/main/variables.tf new file mode 100644 index 0000000..2b328ee --- /dev/null +++ b/infrastructure-provisioning/terraform/gcp/main/variables.tf @@ -0,0 +1,55 @@ +variable "credentials" {} + +variable "project_name" {} + +variable "project_tag" {} + +variable "endpoint_tag" {} + +variable "user_tag" {} + +variable "custom_tag" {} + +variable "notebook_name" {} + +variable "region_var" {} + +variable "zone_var" {} + +variable "product_name" {} + +variable "vpc_name" {} + +variable "subnet_name" {} + +variable "fw_ingress" {} + +variable "fw_egress_public" {} + +variable "fw_egress_private" {} + +variable "network_tag" {} + +variable "sa_email" {} + +variable "cidr_range" {} + +variable "traefik_cidr" {} + +variable "ami" {} + +variable "machine_type" {} + +variable "ssh_key" {} + +variable "gpu_accelerator" {} + +variable "cluster_name" {} + +variable "total_count" {} + +variable "master_shape" {} + +variable "slave_shape" {} + +variable "preemptible_count" {} diff --git a/infrastructure-provisioning/terraform/gcp/modules/common/iam.tf b/infrastructure-provisioning/terraform/gcp/modules/common/iam.tf new file mode 100644 index 0000000..6931f60 --- /dev/null +++ b/infrastructure-provisioning/terraform/gcp/modules/common/iam.tf @@ -0,0 +1,36 @@ +locals { + service_name = "${var.project_tag}-ps-sa" + role_name = "${var.project_tag}-ps-role" +} + +resource "google_service_account" "ps_sa" { + #Create service account for notebooks and computational resources + account_id = "${var.project_tag}-ps-sa" + display_name = "${var.project_tag}-ps-sa" +} + +resource "google_service_account_key" "ps_sa_key" { + #Create service account key + depends_on = ["google_project_iam_member.iam"] + service_account_id = google_service_account.ps_sa.name +} + +resource "google_project_iam_custom_role" "ps-custom-role" { + #Create custom role for ps_sa + role_id = "${replace("${var.project_tag}-ps-role", "-", "_")}" + title = "${var.project_tag}-ps-role" + permissions = "${var.ps_policy}" +} + +resource "google_project_iam_member" "role_for_member" { + #Grant the custom role for the ps_sa + member = "serviceAccount:${google_service_account.ps_sa.email}" + role = "${google_project_iam_custom_role.ps-custom-role.id}" +} + +resource "google_project_iam_member" "iam" { + #Grant other roles for the ps_sa + count = "${length(var.ps_roles)}" + member = "serviceAccount:${google_service_account.ps_sa.email}" + role = "${element(var.ps_roles, count.index)}" +} \ No newline at end of file diff --git a/infrastructure-provisioning/terraform/gcp/modules/common/network.tf b/infrastructure-provisioning/terraform/gcp/modules/common/network.tf new file mode 100644 index 0000000..8d34249 --- /dev/null +++ b/infrastructure-provisioning/terraform/gcp/modules/common/network.tf @@ -0,0 +1,39 @@ +resource "google_compute_subnetwork" "subnet" { + name = "${var.project_tag}-subnet" + ip_cidr_range = "${var.cidr_range}" + region = "${var.region}" + network = "${var.vpc_name}" +} + +resource "google_compute_firewall" "fw_ingress" { + name = "${var.fw_ingress}" + network = "${var.vpc_name}" + allow { + protocol = "all" + } + target_tags = ["${var.network_tag}"] + source_ranges = ["${var.cidr_range}", "${var.traefik_cidr}"] +} + +resource "google_compute_firewall" "fw_egress_public" { + name = "${var.fw_egress_public}" + network = "${var.vpc_name}" + direction = "EGRESS" + allow { + protocol = "tcp" + ports = ["443"] + } + target_tags = ["${var.network_tag}"] + destination_ranges = ["0.0.0.0/0"] +} + +resource "google_compute_firewall" "fw_egress_private" { + name = "${var.fw_egress_private}" + network = "${var.vpc_name}" + direction = "EGRESS" + allow { + protocol = "all" + } + target_tags = ["${var.network_tag}"] + destination_ranges = ["${var.cidr_range}", "${var.traefik_cidr}"] +} \ No newline at end of file diff --git a/infrastructure-provisioning/terraform/gcp/modules/common/variables.tf b/infrastructure-provisioning/terraform/gcp/modules/common/variables.tf new file mode 100644 index 0000000..b4180ad --- /dev/null +++ b/infrastructure-provisioning/terraform/gcp/modules/common/variables.tf @@ -0,0 +1,39 @@ +variable "project_tag" {} + +variable "endpoint_tag" {} + +variable "user_tag" {} + +variable "custom_tag" {} + +variable "region" {} + +variable "product" {} + +variable "vpc_name" {} + +variable "fw_ingress" {} + +variable "fw_egress_public" {} + +variable "fw_egress_private" {} + +variable "network_tag" {} + +variable "cidr_range" {} + +variable "traefik_cidr" {} + +variable "ps_roles" { + type = "list" + default = [ + "roles/dataproc.worker" + ] +} + +variable "ps_policy" { + type = "list" + default = [ + + ] +} \ No newline at end of file diff --git a/infrastructure-provisioning/terraform/gcp/modules/data_engine/instance.tf b/infrastructure-provisioning/terraform/gcp/modules/data_engine/instance.tf new file mode 100644 index 0000000..fdb6250 --- /dev/null +++ b/infrastructure-provisioning/terraform/gcp/modules/data_engine/instance.tf @@ -0,0 +1,101 @@ +locals { + notebook_name = "${var.project_tag}-nb-${var.notebook_name}" + cluster_name = "${var.project_tag}-de-${var.notebook_name}-${var.cluster_name}" +} + +resource "google_compute_instance" "master" { + name = "${local.cluster_name}-m" + machine_type = "${var.master_shape}" + tags = ["${var.network_tag}"] + zone = "${var.zone_var}" + + boot_disk { + initialize_params { + image = "${var.ami}" + size = 30 + } + } + + labels = { + name = "${local.cluster_name}-m" + notebook_name = "${local.notebook_name}" + project = "${var.project_tag}" + product = "${var.product}" + type = "master" + user = "${var.user_tag}" + } + + metadata = { + ssh-keys = "ubuntu:${file("${var.ssh_key}")}" + } + + service_account { + email = "${var.sa_email}" + scopes = ["https://www.googleapis.com/auth/cloud-platform";, "https://www.googleapis.com/auth/compute";] + } + + network_interface { + network = "${var.vpc_name}" + subnetwork = "${var.subnet_name}" + } + + guest_accelerator { + count = "${var.gpu_accelerator != "false" ? 1 : 0}" + type = "nvidia-tesla-k80" + } + + scheduling { + on_host_maintenance = "${var.gpu_accelerator != "false" ? "TERMINATE" : "MIGRATE"}" + } + +} + + +resource "google_compute_instance" "slave" { + count = "${var.total_count - 1}" + name = "${local.cluster_name}-s${count.index + 1}" + machine_type = "${var.slave_shape}" + tags = ["${var.network_tag}"] + zone = "${var.zone_var}" + + boot_disk { + initialize_params { + image = "${var.ami}" + size = 30 + } + } + + labels = { + name = "${local.cluster_name}-s${count.index + 1}" + notebook_name = "${local.notebook_name}" + project = "${var.project_tag}" + product = "${var.product}" + sbn = "${var.project_tag}" + type = "slave" + user = "${var.user_tag}" + } + + metadata = { + ssh-keys = "ubuntu:${file("${var.ssh_key}")}" + } + + service_account { + email = "${var.sa_email}" + scopes = ["https://www.googleapis.com/auth/cloud-platform";, "https://www.googleapis.com/auth/compute";] + } + + network_interface { + network = "${var.vpc_name}" + subnetwork = "${var.subnet_name}" + } + + guest_accelerator { + count = "${var.gpu_accelerator != "false" ? 1 : 0}" + type = "nvidia-tesla-k80" + } + + scheduling { + on_host_maintenance = "${var.gpu_accelerator != "false" ? "TERMINATE" : "MIGRATE"}" + } + +} \ No newline at end of file diff --git a/infrastructure-provisioning/terraform/gcp/modules/data_engine/variables.tf b/infrastructure-provisioning/terraform/gcp/modules/data_engine/variables.tf new file mode 100644 index 0000000..12c4f92 --- /dev/null +++ b/infrastructure-provisioning/terraform/gcp/modules/data_engine/variables.tf @@ -0,0 +1,35 @@ +variable "project_tag" {} + +variable "endpoint_tag" {} + +variable "user_tag" {} + +variable "custom_tag" {} + +variable "product" {} + +variable "notebook_name" {} + +variable "zone_var" {} + +variable "vpc_name" {} + +variable "subnet_name" {} + +variable "network_tag" {} + +variable "sa_email" {} + +variable "ami" {} + +variable "ssh_key" {} + +variable "gpu_accelerator" {} + +variable "cluster_name" {} + +variable "total_count" {} + +variable "master_shape" {} + +variable "slave_shape" {} \ No newline at end of file diff --git a/infrastructure-provisioning/terraform/gcp/modules/dataproc/instance.tf b/infrastructure-provisioning/terraform/gcp/modules/dataproc/instance.tf new file mode 100644 index 0000000..415479c --- /dev/null +++ b/infrastructure-provisioning/terraform/gcp/modules/dataproc/instance.tf @@ -0,0 +1,42 @@ +locals { + dataproc_name = "${var.project_tag}-des-${var.notebook_name}-${var.cluster_name}" +} + +resource "google_dataproc_cluster" "dataproc" { + name = "${local.dataproc_name}" + region = "${var.region}" + labels = { + computational_name = "${var.cluster_name}" + name = "${local.dataproc_name}" + sbn = "${var.project_tag}" + user = "${var.user_tag}" + } + + cluster_config { + + master_config { + num_instances = 1 + machine_type = "${var.master_shape}" + disk_config { + boot_disk_size_gb = 30 + } + } + + worker_config { + num_instances = "${var.total_count - 1}" + machine_type = "${var.slave_shape}" + disk_config { + boot_disk_size_gb = 30 + } + } + + gce_cluster_config { + subnetwork = "${var.subnet_name}" + tags = ["${var.network_tag}"] + } + + preemptible_worker_config { + num_instances = "${var.preemptible_count}" + } + } + } \ No newline at end of file diff --git a/infrastructure-provisioning/terraform/gcp/modules/dataproc/variables.tf b/infrastructure-provisioning/terraform/gcp/modules/dataproc/variables.tf new file mode 100644 index 0000000..d3dcf15 --- /dev/null +++ b/infrastructure-provisioning/terraform/gcp/modules/dataproc/variables.tf @@ -0,0 +1,39 @@ +variable "region" {} + +variable "project_tag" {} + +variable "endpoint_tag" {} + +variable "user_tag" {} + +variable "custom_tag" {} + +variable "product" {} + +variable "notebook_name" {} + +variable "zone_var" {} + +variable "vpc_name" {} + +variable "subnet_name" {} + +variable "network_tag" {} + +variable "sa_email" {} + +variable "ami" {} + +variable "ssh_key" {} + +variable "gpu_accelerator" {} + +variable "cluster_name" {} + +variable "total_count" {} + +variable "master_shape" {} + +variable "slave_shape" {} + +variable "preemptible_count" {} \ No newline at end of file diff --git a/infrastructure-provisioning/terraform/gcp/modules/notebook/instance.tf b/infrastructure-provisioning/terraform/gcp/modules/notebook/instance.tf new file mode 100644 index 0000000..0943e51 --- /dev/null +++ b/infrastructure-provisioning/terraform/gcp/modules/notebook/instance.tf @@ -0,0 +1,65 @@ +locals { + name = "${var.project_tag}-nb-${var.notebook_name}" +} + +resource "google_compute_disk" "secondary" { + name = "${local.name}-secondary" + zone = "${var.zone_var}" + labels = { + name = "${local.name}" + product = "${var.product}" + project = "${var.project_tag}" + user = "${var.user_tag}" + } + physical_block_size_bytes = 4096 + size = 30 +} + +resource "google_compute_instance" "notebook" { + name = "${local.name}" + machine_type = "${var.machine_type}" + tags = ["${var.network_tag}"] + zone = "${var.zone_var}" + + boot_disk { + initialize_params { + image = "${var.ami}" + size = 12 + } + } + + attached_disk { + source = "${google_compute_disk.secondary.self_link}" + } + + labels = { + name = "${local.name}" + product = "${var.product}" + project = "${var.project_tag}" + user = "${var.user_tag}" + } + + metadata = { + ssh-keys = "ubuntu:${file("${var.ssh_key}")}" + } + + service_account { + email = "${var.sa_email}" + scopes = ["https://www.googleapis.com/auth/cloud-platform";, "https://www.googleapis.com/auth/compute";] + } + + network_interface { + network = "${var.vpc_name}" + subnetwork = "${var.subnet_name}" + } + + guest_accelerator { + count = "${var.gpu_accelerator != "false" ? 1 : 0}" + type = "nvidia-tesla-k80" + } + + scheduling { + on_host_maintenance = "${var.gpu_accelerator != "false" ? "TERMINATE" : "MIGRATE"}" + } + +} \ No newline at end of file diff --git a/infrastructure-provisioning/terraform/gcp/modules/notebook/variables.tf b/infrastructure-provisioning/terraform/gcp/modules/notebook/variables.tf new file mode 100644 index 0000000..6217480 --- /dev/null +++ b/infrastructure-provisioning/terraform/gcp/modules/notebook/variables.tf @@ -0,0 +1,29 @@ +variable "project_tag" {} + +variable "endpoint_tag" {} + +variable "user_tag" {} + +variable "custom_tag" {} + +variable "product" {} + +variable "notebook_name" {} + +variable "zone_var" {} + +variable "vpc_name" {} + +variable "subnet_name" {} + +variable "network_tag" {} + +variable "sa_email" {} + +variable "ami" {} + +variable "machine_type" {} + +variable "ssh_key" {} + +variable "gpu_accelerator" {} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
