This is an automated email from the ASF dual-hosted git repository.
damondouglas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 25191d5187d Terraform module to provision a kafka proxy (#31142)
25191d5187d is described below
commit 25191d5187dbb4dd8dd75a62d3cff8f7297fff2e
Author: Damon <[email protected]>
AuthorDate: Thu May 2 11:15:24 2024 -0700
Terraform module to provision a kafka proxy (#31142)
* IaC to provision a kafka proxy
* Install numpy
---
.test-infra/kafka/proxy/.terraform.lock.hcl | 40 +++++++++++++
.test-infra/kafka/proxy/README.md | 56 +++++++++++++++++
.test-infra/kafka/proxy/common.tfvars | 21 +++++++
.test-infra/kafka/proxy/compute.tf | 88 +++++++++++++++++++++++++++
.test-infra/kafka/proxy/prerequisites.tf | 75 +++++++++++++++++++++++
.test-infra/kafka/proxy/provider.tf | 22 +++++++
.test-infra/kafka/proxy/variables.tf | 93 +++++++++++++++++++++++++++++
7 files changed, 395 insertions(+)
diff --git a/.test-infra/kafka/proxy/.terraform.lock.hcl
b/.test-infra/kafka/proxy/.terraform.lock.hcl
new file mode 100644
index 00000000000..fd927e0920a
--- /dev/null
+++ b/.test-infra/kafka/proxy/.terraform.lock.hcl
@@ -0,0 +1,40 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/google" {
+ version = "5.27.0"
+ hashes = [
+ "h1:chdLlH3DV0DXSfV40ZeiJQ+mB+OYt0RMRlpeTsbhHiw=",
+ "zh:08301af898c1a78e78ad547482d50c95a43ef65d09fd5058800cf32cd9c8cd53",
+ "zh:1a4f9e5134e990132978e78ea15431d32e06bf8024fd6733a98faa811ae03efb",
+ "zh:383e66659d69dc4b4a1ad5d7cbc6aa4ce75015f380cfb5f47beaeb506c9e2e1c",
+ "zh:3aa4aff7dd9240fb387271dc791e084d010044dc58336a7a690b0f1a8890ab68",
+ "zh:4084b9a61e662bdd79d1304432dffc6cd3cf00021b937b01001ae9fee5727b12",
+ "zh:448f5d281cab53caacb8759fcd3309c7aa1ba5a210d1866b28e8bd77fd4634ab",
+ "zh:75457a1f0b77bc7477efe58e7b223649340147fd735ed8b8fe57a06ec8459c95",
+ "zh:7648c6ea04d5b1d1413cce880ed77bd7373aef1a58cd5a26394edf64dc6cac11",
+ "zh:b43630367e29a4c185d3eab8b3f84f818e8a91f16007f0e81d876ab96af4ee43",
+ "zh:b478e7d36c5e99f0c026cb05c06047ce1f24fc07284692a10e74214a853e7139",
+ "zh:e6f349125299401049f64e608b3d73236b139e960816fffdd208d1ba405e1804",
+ "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+ ]
+}
+
+provider "registry.terraform.io/hashicorp/random" {
+ version = "3.6.1"
+ hashes = [
+ "h1:1OlP753r4lOKlBprL0HdZGWerm5DCabD5Mli8k8lWAg=",
+ "zh:2a0ec154e39911f19c8214acd6241e469157489fc56b6c739f45fbed5896a176",
+ "zh:57f4e553224a5e849c99131f5e5294be3a7adcabe2d867d8a4fef8d0976e0e52",
+ "zh:58f09948c608e601bd9d0a9e47dcb78e2b2c13b4bda4d8f097d09152ea9e91c5",
+ "zh:5c2a297146ed6fb3fe934c800e78380f700f49ff24dbb5fb5463134948e3a65f",
+ "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+ "zh:7ce41e26f0603e31cdac849085fc99e5cd5b3b73414c6c6d955c0ceb249b593f",
+ "zh:8c9e8d30c4ef08ee8bcc4294dbf3c2115cd7d9049c6ba21422bd3471d92faf8a",
+ "zh:93e91be717a7ffbd6410120eb925ebb8658cc8f563de35a8b53804d33c51c8b0",
+ "zh:982542e921970d727ce10ed64795bf36c4dec77a5db0741d4665230d12250a0d",
+ "zh:b9d1873f14d6033e216510ef541c891f44d249464f13cc07d3f782d09c7d18de",
+ "zh:cfe27faa0bc9556391c8803ade135a5856c34a3fe85b9ae3bdd515013c0c87c1",
+ "zh:e4aabf3184bbb556b89e4b195eab1514c86a2914dd01c23ad9813ec17e863a8a",
+ ]
+}
diff --git a/.test-infra/kafka/proxy/README.md
b/.test-infra/kafka/proxy/README.md
new file mode 100644
index 00000000000..c101b0e3b39
--- /dev/null
+++ b/.test-infra/kafka/proxy/README.md
@@ -0,0 +1,56 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Kafka proxy
+
+Provisions a private IP bastion host on Google Cloud for use as a proxy to a
private IP Kafka instance.
+
+# Prerequisites
+
+- Kafka cluster (See [.test-infra/kafka](..) for available solutions.)
+
+# Usage
+
+## Acquire bootstrap server hosts
+
+One of the variables requires a mapping of bootstrap server hosts to the
desired proxy exposed port. See
+the variable description for `bootstrap_endpoint_mapping` found in the
[variables.tf](variables.tf) file.
+
+## Apply module
+
+Follows typical terraform workflow without the use of a
+[backend](https://developer.hashicorp.com/terraform/language/settings/backends/configuration).
+
+```
+DIR=.test-infra/kafka/proxy
+terraform -chdir=$DIR init
+```
+
+```
+terraform -chdir=$DIR apply -var-file=common.tfvars
-var-file=name_of_your_specific.tfvars
+```
+
+## Invoke gcloud ssh tunneling command
+
+Successful application of the module will output the specific gcloud command
needed to tunnel the kafka traffic
+to your local machine. An example of such output would look similar to:
+
+```
+gcloud compute ssh yourinstance --tunnel-through-iap --project=project
--zone=zone --ssh-flag="-4 -L9093:localhost:9093" --ssh-flag="-4
-L9092:localhost:9092" --ssh-flag="-4 -L9094:localhost:9094"
+```
diff --git a/.test-infra/kafka/proxy/common.tfvars
b/.test-infra/kafka/proxy/common.tfvars
new file mode 100644
index 00000000000..47b03aafb1f
--- /dev/null
+++ b/.test-infra/kafka/proxy/common.tfvars
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+allows_iap_ingress_cidr_range = "35.235.240.0/20"
+kafka_proxy_version = "v0.3.8"
+machine_type = "n1-standard-1"
diff --git a/.test-infra/kafka/proxy/compute.tf
b/.test-infra/kafka/proxy/compute.tf
new file mode 100644
index 00000000000..d673b3bdaed
--- /dev/null
+++ b/.test-infra/kafka/proxy/compute.tf
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+
+ // builds kafka-proxy's bootstrap-server-mapping flag based on the provided
bootstrap_endpoint_mapping variable.
+ bootstrap_server_mapping = join(" ", [
+ for k, v in var.bootstrap_endpoint_mapping
:"--bootstrap-server-mapping=\"${k},0.0.0.0:${v}\""
+ ])
+
+ // builds the gcloud ssh-flag to output after provisioning the bastion host
based on the provided bootstrap_endpoint_mapping variable.
+ ssh_flag = join(" ", [
+ for port in values(var.bootstrap_endpoint_mapping) :"--ssh-flag=\"-4
-L${port}:localhost:${port}\""
+ ])
+}
+
+// Provision a firewall allowing for Identity Aware Proxy (IAP) SSH traffic.
+resource "google_compute_firewall" "allow-ssh-ingress-from-iap" {
+ name =
"${data.google_compute_network.default.name}-allow-ssh-ingress-from-iap"
+ network = data.google_compute_network.default.id
+ direction = "INGRESS"
+ allow {
+ protocol = "TCP"
+ ports = [22]
+ }
+ source_ranges = [var.allows_iap_ingress_cidr_range]
+}
+
+// Generate a random string to use as a postfix for resource naming.
+resource "random_string" "postfix" {
+ length = 4
+ upper = false
+ lower = true
+ numeric = true
+ special = false
+}
+
+// Provision a bastion host for use as a kafka proxy.
+// On launch, installs and executes https://github.com/grepplabs/kafka-proxy.
+resource "google_compute_instance" "bastion" {
+ machine_type = var.machine_type
+ name = "kafka-proxy-${random_string.postfix.result}"
+ tags = ["kafka-proxy"]
+ zone = data.google_compute_zones.available.names[0]
+
+ boot_disk {
+ initialize_params {
+ image = "debian-cloud/debian-12"
+ size = "10"
+ }
+ }
+ // Configures private IP only network interface.
+ network_interface {
+ subnetwork = data.google_compute_subnetwork.default.id
+ }
+ service_account {
+ email = data.google_service_account.default.email
+ // Defer to IAM roles
+ scopes = ["https://www.googleapis.com/auth/cloud-platform";]
+ }
+ metadata_startup_script = <<EOF
+$(gcloud info --format="value(basic.python_location)") -m pip install numpy
+curl -Ls
https://github.com/grepplabs/kafka-proxy/releases/download/${var.kafka_proxy_version}/kafka-proxy-${var.kafka_proxy_version}-linux-amd64.tar.gz
| tar xz
+./kafka-proxy server ${local.bootstrap_server_mapping}
+EOF
+}
+
+// Outputs the gcloud command to tunnel kafka proxy traffic to the local
machine.
+output "gcloud_tunnel_command" {
+ value = <<EOF
+ gcloud compute ssh ${google_compute_instance.bastion.name}
--tunnel-through-iap --project=${var.project}
--zone=${google_compute_instance.bastion.zone} ${local.ssh_flag}
+EOF
+}
diff --git a/.test-infra/kafka/proxy/prerequisites.tf
b/.test-infra/kafka/proxy/prerequisites.tf
new file mode 100644
index 00000000000..2ca0c5a1fda
--- /dev/null
+++ b/.test-infra/kafka/proxy/prerequisites.tf
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "google_project_service" "required" {
+ for_each = toset([
+ "compute",
+ "iam",
+ ])
+ service = "${each.key}.googleapis.com"
+ disable_on_destroy = false
+}
+
+// Query the VPC network to make sure it exists.
+data "google_compute_network" "default" {
+ depends_on = [google_project_service.required]
+ name = var.network
+}
+
+// Query the VPC subnetwork to make sure it exists in the region specified.
+data "google_compute_subnetwork" "default" {
+ depends_on = [google_project_service.required]
+ name = var.subnetwork
+ region = var.region
+ lifecycle {
+ postcondition {
+ condition = self.private_ip_google_access
+ error_message = <<EOT
+fatal: ${self.id} misconfigured: private Google access disabled.
+See https://cloud.google.com/vpc/docs/configure-private-google-access for
details.
+EOT
+ }
+ }
+}
+
+// Query the Google Compute Router to make sure it exists; needed to access
resources outside the VPC network
+// for private nodes.
+data "google_compute_router" "default" {
+ name = var.router
+ network = data.google_compute_network.default.name
+ region = data.google_compute_subnetwork.default.region
+}
+
+// Query the Google Compute Router NAT to make sure it exists.
+data "google_compute_router_nat" "default" {
+ name = var.router_nat
+ router = data.google_compute_router.default.name
+ region = data.google_compute_router.default.region
+}
+
+// Query the Service Account.
+data "google_service_account" "default" {
+ depends_on = [google_project_service.required]
+ account_id = var.service_account_id
+}
+
+// Query available zones in the Compute region.
+data "google_compute_zones" "available" {
+ region = data.google_compute_subnetwork.default.region
+}
+
diff --git a/.test-infra/kafka/proxy/provider.tf
b/.test-infra/kafka/proxy/provider.tf
new file mode 100644
index 00000000000..e070d8408d9
--- /dev/null
+++ b/.test-infra/kafka/proxy/provider.tf
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Setup Google Cloud provider
+provider "google" {
+ project = var.project
+}
diff --git a/.test-infra/kafka/proxy/variables.tf
b/.test-infra/kafka/proxy/variables.tf
new file mode 100644
index 00000000000..343b04c1655
--- /dev/null
+++ b/.test-infra/kafka/proxy/variables.tf
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project" {
+ type = string
+ description = "The Google Cloud Platform (GCP) project within which
resources are provisioned"
+}
+
+variable "region" {
+ type = string
+ description = "The Google Cloud Platform (GCP) region in which to provision
resources"
+}
+
+variable "network" {
+ type = string
+ description = "The Google Cloud Virtual Private Cloud (VPC) network name"
+}
+
+variable "router" {
+ type = string
+ description = "The name of the Google Compute Router resource associated
with the VPC Network"
+}
+
+variable "router_nat" {
+ type = string
+ description = "The name of the NAT associated with the Google Compute Router"
+}
+
+variable "subnetwork" {
+ type = string
+ description = "The Google Cloud Virtual Private Cloud (VPC) subnetwork name"
+}
+
+variable "service_account_id" {
+ type = string
+ description = "The ID of the service account to bind to the bastion host"
+}
+
+variable "kafka_proxy_version" {
+ type = string
+ description = "The release version of
https://github.com/grepplabs/kafka-proxy/releases";
+}
+
+variable "machine_type" {
+ type = string
+ description = "The machine type to specify for the provision bastion host"
+}
+
+variable "bootstrap_endpoint_mapping" {
+ type = map(number)
+ description = <<EOF
+ The mapping of kafka bootstrap server endpoints i.e.
1.2.3.4:9094=0.0.0.0:9092 to assign bootstrap server mapping.
+If Kafka is installed on Kubernetes, run `kubectl get svc` and acquire the
list of load balanced bootstrap IP.
+For example, the output below of `kubectl get svc` would require the
'Configuring as' shown following.
+
+NAME TYPE CLUSTER-IP EXTERNAL-IP PORTS
+kafka-0 LoadBalancer 34.1.2.3 10.1.2.3 tcp-external:9094►12345
+kafka-1 LoadBalancer 34.1.2.4 10.1.2.4 tcp-external:9094►12346
+kafka-2 LoadBalancer 34.1.2.5 10.1.2.5 tcp-external:9094►12347
+
+Configuring as
+{
+ "10.1.2.3:9094": 9092,
+ "10.1.2.4:9094": 9093,
+ "10.1.2.5:9094": 9094,
+}
+
+yields when computing the kafka-proxy flags:
+ 10.1.2.3:9094=0.0.0.0:9092,
+ 10.1.2.4:9094=0.0.0.0:9093,
+ 10.1.2.5:9094=0.0.0.0:9094,
+EOF
+}
+
+variable "allows_iap_ingress_cidr_range" {
+ type = string
+ description = "The CIDR range that identity aware proxy (IAP) uses for TCP
forwarding"
+}
