This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 5bd0c08260e Automation: Tourofbeam backend infrastructure deployment
using cloudbuild (#26504)
5bd0c08260e is described below
commit 5bd0c08260e84d2a5d2fe820950511eae728c58d
Author: ruslan-ikhsan <[email protected]>
AuthorDate: Wed May 31 19:34:30 2023 +0500
Automation: Tourofbeam backend infrastructure deployment using cloudbuild
(#26504)
* Initial commit for TOB Infra via Cloud Build
* Configuring TF for Cloud Build
* Update iam.tf
* Updating TF scripts
* Shell script updates
* Refactoring TF scripts
* Create README.md
* Update README.md
* Update inline.yaml
* Refactoring TF scripts. Renamed yaml to .sh
* Update README.md
* ToB backend and infra triggers
* fixing var issues
* fixing attribure name
* improving readability
* improving readability
* fixing variable names and readability
* fixing variables and env
* adding missing variable
* formatting
* line anding formatting
* fixing test
* fixing gradle to deploy with cloud build
* fixing gradle error
* fix gradle firebase command
* multitenancy for frontend deployment
* adding hosting target
* correct resource dependency
* fixing secret
* missing secret role
* fixing gradle hosting task
* fix gradle errors
* fixing flutter link
* fixing dependency for gradle tasks
* test fix for flutter
* adding conditional logic for hosting site creation
* wrong variable name fixed
* fixing flutter build
* fix regexp
* fix regexp
* fix escape for kotlin
* try another regex
* regex fix
* fixing flutter commands
* fixing flutter
* update hosting target
* fix readme whitespace
* removing unused sa and vars
* missing role to write logs
* fixing role to write to buckets
* updating cb deployment Readme
* switch to ADC authentication for firebase
* disabling firebase token env
* removing firebase token references
* removing incorrect defaults
* removing unused vars
* meaningful name for var
* fixing readme
* release flutter build
* improving ToB CD
* initial deployment of LM
* syntax error fixed
* fixing prod url backend fallback
* fixing LM CD trigger
* fixing git error
* remove html renderer
* adding LM deployment to cd
* update flutter version
* minor formatting
* restoring condition
* updating README
* missing Playground README link added
---------
Co-authored-by: Oleh Borysevych <[email protected]>
---
learning/tour-of-beam/backend/internal/sdk.go | 4 +-
learning/tour-of-beam/backend/internal/sdk_test.go | 1 -
.../backend/samples/api/get_sdk_list.json | 3 +-
learning/tour-of-beam/cloudbuild/01.setup/iam.tf | 76 ++++++++
.../output.tf => cloudbuild/01.setup/provider.tf} | 7 +-
.../01.setup/services.tf} | 13 +-
.../01.setup/terraform.tf} | 13 +-
.../setup => cloudbuild/01.setup}/variables.tf | 20 +-
.../locals.tf => cloudbuild/02.builders/data.tf} | 33 ++--
.../tour-of-beam/cloudbuild/02.builders/locals.tf | 101 ++++++++++
.../02.builders/provider.tf} | 7 +-
.../02.builders/terraform.tf} | 14 +-
.../cloudbuild/02.builders/triggers.tf | 217 +++++++++++++++++++++
.../cloudbuild/02.builders/variables.tf | 132 +++++++++++++
learning/tour-of-beam/cloudbuild/README.md | 153 +++++++++++++++
.../cloudbuild/scripts/tob_deploy_infra_backend.sh | 89 +++++++++
.../tour-of-beam/cloudbuild/scripts/tob_lm_cd.sh | 42 ++++
learning/tour-of-beam/terraform/README.md | 25 +--
learning/tour-of-beam/terraform/build.gradle.kts | 95 ++++++---
.../tour-of-beam/terraform/cloud_functions/main.tf | 2 +-
.../terraform/cloud_functions/variables.tf | 2 +-
.../terraform/functions_buckets/locals.tf | 11 +-
.../terraform/functions_buckets/variables.tf | 4 +
learning/tour-of-beam/terraform/main.tf | 5 +-
learning/tour-of-beam/terraform/setup/iam.tf | 12 +-
learning/tour-of-beam/terraform/setup/locals.tf | 9 +-
learning/tour-of-beam/terraform/setup/output.tf | 2 +-
learning/tour-of-beam/terraform/setup/variables.tf | 6 +-
learning/tour-of-beam/terraform/variables.tf | 9 +-
.../lib/src/constants/backend_urls.dart | 2 +-
30 files changed, 978 insertions(+), 131 deletions(-)
diff --git a/learning/tour-of-beam/backend/internal/sdk.go
b/learning/tour-of-beam/backend/internal/sdk.go
index f088eb3b35e..cf14600ef6c 100644
--- a/learning/tour-of-beam/backend/internal/sdk.go
+++ b/learning/tour-of-beam/backend/internal/sdk.go
@@ -79,8 +79,8 @@ func ParseSdk(s string) Sdk {
}
func MakeSdkList() SdkList {
- sdks := make([]SdkItem, 0, 4)
- for _, sdk := range []Sdk{SDK_JAVA, SDK_PYTHON, SDK_GO, SDK_SCIO} {
+ sdks := make([]SdkItem, 0, 3)
+ for _, sdk := range []Sdk{SDK_JAVA, SDK_PYTHON, SDK_GO} {
sdks = append(sdks, SdkItem{Id: sdk.String(), Title:
sdk.Title()})
}
return SdkList{Sdks: sdks}
diff --git a/learning/tour-of-beam/backend/internal/sdk_test.go
b/learning/tour-of-beam/backend/internal/sdk_test.go
index 1352893c237..3d33aceed74 100644
--- a/learning/tour-of-beam/backend/internal/sdk_test.go
+++ b/learning/tour-of-beam/backend/internal/sdk_test.go
@@ -76,7 +76,6 @@ func TestSdkList(t *testing.T) {
{"java", "Java"},
{"python", "Python"},
{"go", "Go"},
- {"scio", "SCIO"},
},
}, MakeSdkList())
}
diff --git a/learning/tour-of-beam/backend/samples/api/get_sdk_list.json
b/learning/tour-of-beam/backend/samples/api/get_sdk_list.json
index b24d25f1f9e..7c91da96717 100644
--- a/learning/tour-of-beam/backend/samples/api/get_sdk_list.json
+++ b/learning/tour-of-beam/backend/samples/api/get_sdk_list.json
@@ -2,7 +2,6 @@
"sdks" : [
{"id": "java", "title": "Java"},
{"id": "python", "title": "Python"},
- {"id": "go", "title": "Go"},
- {"id": "scio", "title": "SCIO"}
+ {"id": "go", "title": "Go"}
]
}
\ No newline at end of file
diff --git a/learning/tour-of-beam/cloudbuild/01.setup/iam.tf
b/learning/tour-of-beam/cloudbuild/01.setup/iam.tf
new file mode 100644
index 00000000000..3f619dc7073
--- /dev/null
+++ b/learning/tour-of-beam/cloudbuild/01.setup/iam.tf
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Create cloud build service account
+resource "google_service_account" "tourofbeam_deploy_sa" {
+ account_id = var.tourofbeam_deploy_sa
+ description = "The service account to be used by cloud build to deploy Tour
of Beam backend"
+}
+
+resource "google_service_account" "tourofbeam_ci_sa" {
+ account_id = var.tourofbeam_ci_sa
+ description = "The service account to be used by cloud build to run CI
checks for Tour of Beam backend"
+}
+
+resource "google_service_account" "tourofbeam_cd_sa" {
+ account_id = var.tourofbeam_cd_sa
+ description = "The service account to be used by cloud build to run CD
checks for Tour of Beam backend"
+}
+
+# Assign IAM roles to cloud build service account
+resource "google_project_iam_member" "tourofbeam_backend_deployer_roles" {
+ for_each = toset([
+ "roles/datastore.indexAdmin",
+ "roles/datastore.user",
+ "roles/iam.serviceAccountCreator",
+ "roles/iam.serviceAccountUser",
+ "roles/serviceusage.serviceUsageAdmin",
+ "roles/storage.admin",
+ "roles/firebase.admin",
+ "roles/container.clusterViewer",
+ "roles/logging.logWriter",
+ "roles/cloudfunctions.admin"
+ ])
+ role = each.key
+ member =
"serviceAccount:${google_service_account.tourofbeam_deploy_sa.email}"
+ project = var.project_id
+}
+
+resource "google_project_iam_member" "tourofbeam_ci_sa_roles" {
+ for_each = toset([
+ "roles/secretmanager.secretAccessor",
+ "roles/storage.insightsCollectorService",
+ "roles/storage.objectAdmin"
+ ])
+ role = each.key
+ member = "serviceAccount:${google_service_account.tourofbeam_ci_sa.email}"
+ project = var.project_id
+}
+
+resource "google_project_iam_member" "tourofbeam_cd_sa_roles" {
+ for_each = toset([
+ "roles/datastore.user",
+ "roles/storage.objectAdmin",
+ "roles/secretmanager.secretAccessor",
+ "roles/storage.insightsCollectorService"
+
+ ])
+ role = each.key
+ member = "serviceAccount:${google_service_account.tourofbeam_cd_sa.email}"
+ project = var.project_id
+}
+
diff --git a/learning/tour-of-beam/terraform/setup/output.tf
b/learning/tour-of-beam/cloudbuild/01.setup/provider.tf
similarity index 83%
copy from learning/tour-of-beam/terraform/setup/output.tf
copy to learning/tour-of-beam/cloudbuild/01.setup/provider.tf
index bd069f1ed8d..88041d610b3 100644
--- a/learning/tour-of-beam/terraform/setup/output.tf
+++ b/learning/tour-of-beam/cloudbuild/01.setup/provider.tf
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-# Output used to assign service account to cloud functions
-output "service-account-email" {
- value = google_service_account.cloud_function_sa.email
-}
\ No newline at end of file
+provider "google" {
+ project = var.project_id
+}
diff --git a/learning/tour-of-beam/terraform/functions_buckets/variables.tf
b/learning/tour-of-beam/cloudbuild/01.setup/services.tf
similarity index 75%
copy from learning/tour-of-beam/terraform/functions_buckets/variables.tf
copy to learning/tour-of-beam/cloudbuild/01.setup/services.tf
index f1cb931f2a2..635b2122f23 100644
--- a/learning/tour-of-beam/terraform/functions_buckets/variables.tf
+++ b/learning/tour-of-beam/cloudbuild/01.setup/services.tf
@@ -15,7 +15,14 @@
# specific language governing permissions and limitations
# under the License.
-# GCP region where GCS bucket will be created
-variable "region" {
- description = "The GCP region where GCS bucket will be created (For Cloud
Functions source code)"
+# Enable required GCP APIs
+resource "google_project_service" "required_services" {
+ for_each = toset([
+ "cloudresourcemanager",
+ "iam",
+ "cloudbuild",
+ "cloudfunctions"
+ ])
+ service = "${each.key}.googleapis.com"
+ disable_on_destroy = false
}
diff --git a/learning/tour-of-beam/terraform/functions_buckets/variables.tf
b/learning/tour-of-beam/cloudbuild/01.setup/terraform.tf
similarity index 82%
copy from learning/tour-of-beam/terraform/functions_buckets/variables.tf
copy to learning/tour-of-beam/cloudbuild/01.setup/terraform.tf
index f1cb931f2a2..0c9748b8666 100644
--- a/learning/tour-of-beam/terraform/functions_buckets/variables.tf
+++ b/learning/tour-of-beam/cloudbuild/01.setup/terraform.tf
@@ -15,7 +15,14 @@
# specific language governing permissions and limitations
# under the License.
-# GCP region where GCS bucket will be created
-variable "region" {
- description = "The GCP region where GCS bucket will be created (For Cloud
Functions source code)"
+terraform {
+ backend "gcs" {
+ prefix = "01.setup"
+ }
+ required_providers {
+ google = {
+ source = "hashicorp/google"
+ version = "~> 4.62.0"
+ }
+ }
}
diff --git a/learning/tour-of-beam/terraform/setup/variables.tf
b/learning/tour-of-beam/cloudbuild/01.setup/variables.tf
similarity index 61%
copy from learning/tour-of-beam/terraform/setup/variables.tf
copy to learning/tour-of-beam/cloudbuild/01.setup/variables.tf
index dcb6f54808b..c1dc5f64c6d 100644
--- a/learning/tour-of-beam/terraform/setup/variables.tf
+++ b/learning/tour-of-beam/cloudbuild/01.setup/variables.tf
@@ -15,13 +15,23 @@
# specific language governing permissions and limitations
# under the License.
-# Required and not inferred from the provider argument.
-# Generated by command (gcloud config get-value project) in Kotlin Gradle
script
variable "project_id" {
description = "The ID of the Google Cloud project within which resources are
provisioned"
}
-# This variable is generated by command (gcloud config get-value core/account)
in Kotlin Gradle script
-variable "gcloud_init_account" {
- description = "User Account ID logged in with gcloud init command (e.g.
[email protected])"
+variable "tourofbeam_deploy_sa" {
+ description = "The ID of the cloud build service account responsible for
deploying the Tour of Beam"
+ default = "tourofbeam-cloudbuild-deploy-sa"
}
+
+variable "tourofbeam_ci_sa" {
+ description = "The ID of the cloud build service account responsible for
running Tour of Beam CI checks and scripts"
+ default = "tourofbeam-cloudbuild-ci-sa"
+}
+
+variable "tourofbeam_cd_sa" {
+ description = "The ID of the cloud build service account responsible for
running Tour of Beam CD checks and scripts"
+ default = "tourofbeam-cloudbuild-cd-sa"
+}
+
+
diff --git a/learning/tour-of-beam/terraform/setup/locals.tf
b/learning/tour-of-beam/cloudbuild/02.builders/data.tf
similarity index 51%
copy from learning/tour-of-beam/terraform/setup/locals.tf
copy to learning/tour-of-beam/cloudbuild/02.builders/data.tf
index 9817b8f2821..340ad972d42 100644
--- a/learning/tour-of-beam/terraform/setup/locals.tf
+++ b/learning/tour-of-beam/cloudbuild/02.builders/data.tf
@@ -15,20 +15,27 @@
# specific language governing permissions and limitations
# under the License.
-# Local value to store generated Cloud Functions' Service account name
+# Takes data of service account created for cloud build trigger
+ data "google_service_account" "tourofbeam_deployer" {
+ account_id = var.tourofbeam_deploy_sa
+ }
+
+ data "google_service_account" "tourofbeam_ci_runner" {
+ account_id = var.tourofbeam_ci_sa
+ }
-resource "random_string" "id" {
- length = 4
- upper = false
- special = false
-}
+ data "google_service_account" "tourofbeam_cd_runner" {
+ account_id = var.tourofbeam_cd_sa
+ }
-variable "resource_name_prefix" {
- type = string
- description = "The resource name prefix applied to all resource naming for
the application"
- default = "tour-of-beam"
+# Takes data of secretst created for cloud build trigger
+ data "google_secret_manager_secret_version"
"secret_webhook_cloudbuild_trigger_cicd_data" {
+ secret = var.webhook_trigger_secret_id
+ version = "latest"
}
-locals {
- cloudfunctions_service_account =
"${var.resource_name_prefix}-cf-sa-${random_string.id.result}"
-}
\ No newline at end of file
+ data "google_secret_manager_secret_version" "secret_gh_pat_cloudbuild_data" {
+ secret = var.gh_pat_secret_id
+ version = "latest"
+}
+
\ No newline at end of file
diff --git a/learning/tour-of-beam/cloudbuild/02.builders/locals.tf
b/learning/tour-of-beam/cloudbuild/02.builders/locals.tf
new file mode 100644
index 00000000000..946641c0beb
--- /dev/null
+++ b/learning/tour-of-beam/cloudbuild/02.builders/locals.tf
@@ -0,0 +1,101 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+locals {
+ cloudbuild_init_environment = [
+ "BRANCH_NAME=$_BRANCH_NAME",
+ "REPO_NAME=$_REPO_NAME" ,
+ "PG_REGION=$_PG_REGION",
+ "DNS_NAME=$_DNS_NAME",
+ "WEB_APP_ID=$_WEB_APP_ID",
+ "PG_GKE_ZONE=$_PG_GKE_ZONE",
+ "PG_GKE_NAME=$_PG_GKE_NAME",
+ "PROJECT_ID=$PROJECT_ID",
+ "STATE_BUCKET=$_STATE_BUCKET",
+
+ # Learning material
+ "DATASTORE_PROJECT_ID=$PROJECT_ID",
+ "DATASTORE_NAMESPACE=$_PG_DATASTORE_NAMESPACE",
+ "TOB_LEARNING_ROOT=$_TOB_LEARNING_ROOT",
+
+ # Terraform variables
+ "TF_VAR_environment=$_ENVIRONMENT_NAME",
+ "TF_VAR_region=$_PG_REGION",
+ "TF_VAR_project_id=$PROJECT_ID",
+ "TF_VAR_datastore_namespace=$_PG_DATASTORE_NAMESPACE",
+ ]
+
+ cloudbuild_cd_environment = [
+ "PROJECT_ID=$PROJECT_ID",
+ "DATASTORE_NAMESPACE=$_DATASTORE_NAMESPACE",
+ "DNS_NAME=$_DNS_NAME",
+ "PR_URL=$_PR_URL",
+ "TARGET_PR_REPO_BRANCH=$_TARGET_PR_REPO_BRANCH",
+ "PR_TYPE=$_PR_TYPE",
+ "MERGE_STATUS=$_MERGE_STATUS",
+ "MERGE_COMMIT=$_MERGE_COMMIT",
+ "ORIGIN=$_ORIGIN",
+ "SUBDIRS=$_SUBDIRS",
+ "SDKS=$_SDKS",
+ "BEAM_CONCURRENCY=$_BEAM_CONCURRENCY",
+ "PR_COMMIT=$_PR_COMMIT",
+
"CD_SCRIPT_PATH=beam/playground/infrastructure/cloudbuild/playground_cd_examples.sh",
+ "FORCE_CD=false",
+ # Learning material
+ "DATASTORE_PROJECT_ID=$PROJECT_ID",
+ "DATASTORE_NAMESPACE=$_DATASTORE_NAMESPACE",
+ "TOB_LEARNING_ROOT=$_TOB_LEARNING_ROOT",
+ ]
+
+ cloudbuild_cd_environment_manual = [
+ "PROJECT_ID=$PROJECT_ID",
+ "DNS_NAME=$_DNS_NAME",
+ "PR_URL=URL",
+ "TARGET_PR_REPO_BRANCH=apache:master",
+ "PR_TYPE=closed",
+ "MERGE_STATUS=true",
+ "MERGE_COMMIT=$_MERGE_COMMIT",
+ "ORIGIN=$_ORIGIN",
+ "SUBDIRS=$_SUBDIRS",
+ "SDKS=$_SDKS",
+ "BEAM_CONCURRENCY=$_BEAM_CONCURRENCY",
+ "PR_COMMIT=$_PR_COMMIT",
+
"CD_SCRIPT_PATH=beam/playground/infrastructure/cloudbuild/playground_cd_examples.sh",
+ "FORCE_CD=true",
+ # Learning material
+ "DATASTORE_PROJECT_ID=$PROJECT_ID",
+ "DATASTORE_NAMESPACE=$_DATASTORE_NAMESPACE",
+ "TOB_LEARNING_ROOT=$_TOB_LEARNING_ROOT",
+ ]
+
+ cloudbuild_ci_environment = [
+ "PROJECT_ID=$PROJECT_ID",
+ "PR_BRANCH=$_PR_BRANCH",
+ "PR_URL=$_PR_URL",
+ "PR_TYPE=$_PR_TYPE",
+ "PR_COMMIT=$_PR_COMMIT",
+ "PR_NUMBER=$_PR_NUMBER",
+
"CI_SCRIPT_PATH=beam/playground/infrastructure/cloudbuild/playground_ci_examples.sh",
+ "PUBLIC_BUCKET=$_PUBLIC_BUCKET",
+ "PUBLIC_LOG=$_PUBLIC_LOG",
+ "PUBLIC_LOG_URL=$_PUBLIC_LOG_URL",
+ "PUBLIC_LOG_LOCAL=$_PUBLIC_LOG_LOCAL",
+ "FORK_REPO=$_FORK_REPO",
+ "BASE_REF=$_BASE_REF",
+ "BEAM_VERSION=$_BEAM_VERSION"
+ ]
+}
diff --git a/learning/tour-of-beam/terraform/setup/output.tf
b/learning/tour-of-beam/cloudbuild/02.builders/provider.tf
similarity index 83%
copy from learning/tour-of-beam/terraform/setup/output.tf
copy to learning/tour-of-beam/cloudbuild/02.builders/provider.tf
index bd069f1ed8d..88041d610b3 100644
--- a/learning/tour-of-beam/terraform/setup/output.tf
+++ b/learning/tour-of-beam/cloudbuild/02.builders/provider.tf
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-# Output used to assign service account to cloud functions
-output "service-account-email" {
- value = google_service_account.cloud_function_sa.email
-}
\ No newline at end of file
+provider "google" {
+ project = var.project_id
+}
diff --git a/learning/tour-of-beam/terraform/functions_buckets/variables.tf
b/learning/tour-of-beam/cloudbuild/02.builders/terraform.tf
similarity index 77%
copy from learning/tour-of-beam/terraform/functions_buckets/variables.tf
copy to learning/tour-of-beam/cloudbuild/02.builders/terraform.tf
index f1cb931f2a2..e5ba50c7a15 100644
--- a/learning/tour-of-beam/terraform/functions_buckets/variables.tf
+++ b/learning/tour-of-beam/cloudbuild/02.builders/terraform.tf
@@ -15,7 +15,15 @@
# specific language governing permissions and limitations
# under the License.
-# GCP region where GCS bucket will be created
-variable "region" {
- description = "The GCP region where GCS bucket will be created (For Cloud
Functions source code)"
+# Using 4.62.0 provider to run script block in build
+terraform {
+ backend "gcs" {
+ prefix = "02.builders"
+ }
+ required_providers {
+ google = {
+ source = "hashicorp/google"
+ version = "~> 4.62.0"
+ }
+ }
}
diff --git a/learning/tour-of-beam/cloudbuild/02.builders/triggers.tf
b/learning/tour-of-beam/cloudbuild/02.builders/triggers.tf
new file mode 100644
index 00000000000..00c1c7a6c87
--- /dev/null
+++ b/learning/tour-of-beam/cloudbuild/02.builders/triggers.tf
@@ -0,0 +1,217 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the```
+# specific language governing permissions and limitations
+# under the License.
+
+# This creates cloud build trigger for deploying Tour of Beam backend
+resource "google_cloudbuild_trigger" "tourofbeam_deployment_trigger" {
+ name = var.tourofbeam_deploy_trigger_name
+
+ description = "Trigger used to deploy Tour of Beam infrastructure and
backend"
+
+ source_to_build {
+ uri = var.trigger_source_repo
+ ref = "refs/heads/${var.trigger_source_branch}"
+ repo_type = "GITHUB"
+ }
+
+ build {
+ timeout = "7200s"
+ options {
+ machine_type = var.cloudbuild_machine_type
+ logging = "GCS_ONLY"
+ }
+
+ logs_bucket = "gs://${var.tourofbeam_cb_private_bucket}"
+
+ step {
+ id = "Deploy Tour of Beam"
+ script = file("../scripts/tob_deploy_infra_backend.sh")
+ name = "ubuntu"
+ env = local.cloudbuild_init_environment
+ }
+ }
+
+ substitutions = {
+ _PG_REGION = var.pg_region
+ _PG_GKE_ZONE = var.pg_gke_zone
+ _PG_GKE_NAME = var.pg_gke_name
+ _PG_DATASTORE_NAMESPACE = var.pg_datastore_namespace
+ _DNS_NAME = var.playground_dns_name
+ _WEB_APP_ID = var.web_app_id
+ _STATE_BUCKET = var.tourofbeam_deployment_state_bucket
+ _ENVIRONMENT_NAME = var.environment_name
+ _TOB_LEARNING_ROOT = var.tourofbeam_learning_root
+ _BRANCH_NAME = var.terraform_source_branch
+ _REPO_NAME = var.terraform_source_repo
+ }
+
+ service_account = data.google_service_account.tourofbeam_deployer.id
+}
+
+resource "google_cloudbuild_trigger" "tourofbeam_ci_trigger" {
+ name = var.tourofbeam_ci_trigger_name
+ project = var.project_id
+
+ description = "Validate changed examples and set commit status messages in
GitHub"
+
+ service_account = data.google_service_account.tourofbeam_ci_runner.id
+
+ webhook_config {
+ secret =
data.google_secret_manager_secret_version.secret_webhook_cloudbuild_trigger_cicd_data.id
+ }
+
+ build {
+ timeout = "7200s"
+ options {
+ machine_type = var.cloudbuild_machine_type
+ logging = "GCS_ONLY"
+ }
+ logs_bucket = "gs://${var.tourofbeam_cb_private_bucket}"
+ step {
+ id = "Run CI"
+ script =
file("../../../../playground/infrastructure/cloudbuild/cloudbuild_playground_ci_examples.sh")
+ name = "ubuntu"
+ env = local.cloudbuild_ci_environment
+ secret_env = ["PAT"]
+ }
+ available_secrets {
+ secret_manager {
+ env = "PAT"
+ version_name =
"${data.google_secret_manager_secret_version.secret_gh_pat_cloudbuild_data.secret}/versions/latest"
+ }
+ }
+ substitutions = {
+ _PR_BRANCH = "$(body.pull_request.head.ref)"
+ _PR_URL = "$(body.pull_request._links.html.href)"
+ _PR_TYPE = "$(body.action)"
+ _PR_COMMIT = "$(body.pull_request.head.sha)"
+ _PR_NUMBER = "$(body.number)"
+ _FORK_REPO = "$(body.pull_request.head.repo.full_name)"
+ _BASE_REF = "$(body.pull_request.base.ref)"
+ _PUBLIC_LOG_LOCAL: "/tmp/$${_PUBLIC_LOG}"
+ _PUBLIC_LOG: "CI_PR$${_PR_NUMBER}_$${_PR_COMMIT}_$${BUILD_ID}.txt"
+ _PUBLIC_BUCKET: "${var.tourofbeam_cb_public_bucket}"
+ _PUBLIC_LOG_URL:
"https://storage.googleapis.com/$${_PUBLIC_BUCKET}/$${_PUBLIC_LOG}";
+ }
+ }
+
+ substitutions = {
+ _BEAM_VERSION = "2.47.0"
+ }
+
+}
+
+resource "google_cloudbuild_trigger" "tourofbeam_cd_trigger" {
+ name = var.tourofbeam_cd_trigger_name
+ project = var.project_id
+
+ description = "Automatically update examples for an existing Tour of Beam
environment"
+
+ service_account = data.google_service_account.tourofbeam_cd_runner.id
+
+ webhook_config {
+ secret =
data.google_secret_manager_secret_version.secret_webhook_cloudbuild_trigger_cicd_data.id
+ }
+
+ build {
+ timeout = "7200s"
+ options {
+ machine_type = var.cloudbuild_machine_type
+ logging = "GCS_ONLY"
+ }
+ logs_bucket = "gs://${var.tourofbeam_cb_private_bucket}"
+ step {
+ id = "Run Learning Materials CD"
+ script = file("../scripts/tob_lm_cd.sh")
+ name = "ubuntu"
+ env = local.cloudbuild_cd_environment_manual
+ }
+ step {
+ id = "Run Example CD"
+ script =
file("../../../../playground/infrastructure/cloudbuild/cloudbuild_playground_cd_examples.sh")
+ name = "ubuntu"
+ env = local.cloudbuild_cd_environment
+ }
+ substitutions = {
+ _PR_URL = "$(body.pull_request._links.html.href)"
+ _TARGET_PR_REPO_BRANCH = "$(body.pull_request.base.label)"
+ _PR_TYPE = "$(body.action)"
+ _MERGE_STATUS = "$(body.pull_request.merged)"
+ _MERGE_COMMIT = "$(body.pull_request.merge_commit_sha)"
+ }
+ }
+
+ substitutions = {
+ _DNS_NAME = var.playground_dns_name
+ _DATASTORE_NAMESPACE = var.pg_datastore_namespace
+ _ORIGIN = "TB_EXAMPLES"
+ _TOB_LEARNING_ROOT = var.tourofbeam_learning_root
+ _SDKS = "java python go"
+ _SUBDIRS = "./learning/tour-of-beam/learning-content"
+ _BEAM_CONCURRENCY = "4"
+ }
+
+}
+
+resource "google_cloudbuild_trigger" "tourofbeam_cd_manual_trigger" {
+ name = "${var.tourofbeam_cd_trigger_name}-manual"
+ project = var.project_id
+
+ description = "Manually update examples for an existing Tour of Beam
environment"
+
+ service_account = data.google_service_account.tourofbeam_cd_runner.id
+
+ source_to_build {
+ uri = var.trigger_source_repo
+ ref = "refs/heads/${var.trigger_source_branch}"
+ repo_type = "GITHUB"
+ }
+
+ build {
+ timeout = "7200s"
+ options {
+ machine_type = var.cloudbuild_machine_type
+ logging = "GCS_ONLY"
+ }
+ logs_bucket = "gs://${var.tourofbeam_cb_private_bucket}"
+
+ step {
+ id = "Run Learning Materials CD"
+ script = file("../scripts/tob_lm_cd.sh")
+ name = "ubuntu"
+ env = local.cloudbuild_cd_environment_manual
+ }
+ step {
+ id = "Run Example CD"
+ script =
file("../../../../playground/infrastructure/cloudbuild/cloudbuild_playground_cd_examples.sh")
+ name = "ubuntu"
+ env = local.cloudbuild_cd_environment_manual
+ }
+ }
+
+ substitutions = {
+ _DNS_NAME = var.playground_dns_name
+ _DATASTORE_NAMESPACE = var.pg_datastore_namespace
+ _MERGE_COMMIT = "master"
+ _ORIGIN = "TB_EXAMPLES"
+ _TOB_LEARNING_ROOT = var.tourofbeam_learning_root
+ _SDKS = "java python go"
+ _SUBDIRS = "./learning/tour-of-beam/learning-content"
+ _BEAM_CONCURRENCY = "4"
+ }
+
+}
+
diff --git a/learning/tour-of-beam/cloudbuild/02.builders/variables.tf
b/learning/tour-of-beam/cloudbuild/02.builders/variables.tf
new file mode 100644
index 00000000000..4b687ab52d0
--- /dev/null
+++ b/learning/tour-of-beam/cloudbuild/02.builders/variables.tf
@@ -0,0 +1,132 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+variable "project_id" {
+ description = "GCP project id where resources will be created"
+}
+
+# Trigger service account variables
+variable "tourofbeam_deploy_sa" {
+ description = "Service account name to be created and used by cloud build
deployer"
+}
+
+variable "tourofbeam_ci_sa" {
+ description = "Service account name to be created and used by cloud build
CI"
+}
+
+variable "tourofbeam_cd_sa" {
+ description = "Service account name to be created and used by cloud build
CD"
+}
+
+# Playground variables
+variable "environment_name" {
+ description = "Environment name for Tour of Beam backend (e.g. prod,
staging). To support multi-environment on same GCP project"
+}
+
+variable "playground_dns_name" {
+ description = "The DNS A-record name for Playground website"
+ default = "fqdn.playground.zone"
+}
+
+variable "pg_region" {
+ description = "Existing Beam Playground's region (e.g: us-west1)"
+}
+
+variable "pg_gke_zone" {
+ description = "Existing Beam Playground GKE cluster's zone (e.g.
us-west1-b)"
+}
+
+variable "pg_gke_name" {
+ description = "Existing Beam Playground GKE cluster's name"
+}
+
+variable "pg_datastore_namespace" {
+ description = "Existing Beam Playground's datastore namespace"
+}
+
+variable "tourofbeam_deployment_state_bucket" {
+ description = "Existing GCS bucket's to store Terraform state for Tour of
Beam deployment"
+}
+
+# Trigger variables
+variable "trigger_source_repo" {
+ default = "https://github.com/beamplayground/deploy-workaround";
+}
+
+variable "trigger_source_branch" {
+ description = "Source branch used for github trigger, not used but reqired
due to cloudbuild limitation"
+ default = "main"
+}
+
+variable "webhook_trigger_secret_id" {
+ description = "The name of the secret for webhook config cloud build trigger
(CI/CD)"
+ default = "playground-cicd-webhook"
+}
+
+variable "gh_pat_secret_id" {
+ description = "The name of the secret for GitHub Personal Access Token.
Required for cloud build trigger (CI/CD)"
+ default = "playground-github-pat-ci"
+}
+
+variable "tourofbeam_deploy_trigger_name" {
+ description = "The name of the trigger to deploy Tour of Beam"
+ default = "TourOfBeam-Deploy"
+}
+
+variable "tourofbeam_ci_trigger_name" {
+ description = "The name of the trigger to run CI checks"
+ default = "TourOfBeam-CI"
+}
+
+variable "tourofbeam_cd_trigger_name" {
+ description = "The name of the trigger to run CD checks"
+ default = "TourOfBeam-CD"
+}
+
+variable "cloudbuild_machine_type" {
+ description = "Machine type used for cloudbuild runtime"
+ default = "E2_HIGHCPU_32"
+}
+
+variable "tourofbeam_cb_private_bucket" {
+ description = "The Google Cloud Platform GCS bucket name for Tour of Beam
Cloudbuild Private logs"
+}
+
+variable "tourofbeam_cb_public_bucket" {
+ description = "The Google Cloud Platform GCS bucket name for Tour of Beam
Cloudbuild Public logs"
+}
+
+variable "terraform_source_repo" {
+ description = "Repo used to fetch terraform code"
+ default = "https://github.com/apache/beam";
+}
+
+variable "terraform_source_branch" {
+ description = "Branch used to fetch terraform code"
+ default = "master"
+}
+
+variable "tourofbeam_learning_root" {
+ description = "Existing Tour of Beam learning material root"
+ default = "../learning-content/"
+}
+
+variable "web_app_id" {
+ description = "Tour of Beam web app id"
+ default = "tourofbeam-web-app"
+}
diff --git a/learning/tour-of-beam/cloudbuild/README.md
b/learning/tour-of-beam/cloudbuild/README.md
new file mode 100644
index 00000000000..cbf09d66493
--- /dev/null
+++ b/learning/tour-of-beam/cloudbuild/README.md
@@ -0,0 +1,153 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Tour of Beam Cloud Build Setup
+
+This directory organizes Infrastructure-as-Code to provision dependent
resources and set up Cloud Build for automated Tour of Beam Backend
Infrastructure provisioning, Backend and frontend deployment, manual and
automated CI/CD.
+
+## Requirements:
+
+1. [GCP
project](https://cloud.google.com/resource-manager/docs/creating-managing-projects)
+2. Existing Beam Playground environment/infrastructure deployed in same GCP
Project. Tour of Beam will be deployed to that same location (region/zone). If
you don't have one, please follow [Beam Playground deployment
instructions](./../../../playground/terraform/infrastructure/cloudbuild-manual-setup/README.md)
to deploy it.
+3. [GCP User
account](https://cloud.google.com/appengine/docs/standard/access-control?tab=python)
_(Note: You will find the instruction "How to create User account" for your
new project)_<br>
+ Ensure that the account has at least the following [IAM
roles](https://cloud.google.com/iam/docs/understanding-roles):
+
+ - Cloud Datastore Owner
+ - Create Service Accounts
+ - Security Admin
+ - Service Account User
+ - Service Usage Admin
+ - Storage Admin
+ - Kubernetes Engine Cluster Viewer
+ - Secret Manager Admin
+ - Cloud Build Editor
+
+4. [Google Cloud Storage
buckets](https://cloud.google.com/storage/docs/creating-buckets) for:
+- Terraform state for Cloud Build triggers: \<tourofbeam-triggers-state-env\>
+- Cloud Build private logs: \<tourofbeam-cb-private-logs-env\>
+- Cloud Build public logs: \<tourofbeam-cb-public-logs-env\>. Don't enforce
public access prevention on this bucket.
+- Terraform state for Tour of Beam deployment:
\<tourofbeam-deployment-state-env\>
+
+It's advised to add environment name to the bucket name to avoid collisions
with other environments.
+
+5. An OS with the following software installed:
+
+* [Terraform](https://www.terraform.io/downloads)
+* [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk)
+* [Kubectl authentication
plugin](https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke)
+* [Git client](https://git-scm.com/downloads)
+
+DEV NOTE: GCP Cloud shell can be used for deployment. It has all required
software pre-installed.
+
+6. [Apache Beam GitHub](https://github.com/apache/beam) repository cloned
locally
+
+# Prepare deployment configuration
+
+1. Generate a Terraform variable file called
`beam/learning/tour-of-beam/cloudbuild/common.tfvars`. Place the values listed
below into the file, adjusting them as needed:
+
+project_id = "apache-beam-testing" # GCP project ID
+tourofbeam_deploy_sa = "tourofbeam-cb-deploy-env" # Service account for
Initialize-Playground-environment trigger
+tourofbeam_ci_sa = "tourofbeam-cb-ci-env" # Service account for CI
trigger
+tourofbeam_cd_sa = "tourofbeam-cb-cd-env" # Service account for CD
trigger
+environment_name = "env" # Environment name
+playground_dns_name = "fqdn.beam.apache.org" # Playground DNS name
+pg_region = "us-west1" # Playground region
+pg_gke_zone = "us-west1-a" # Playground GKE zone
+pg_gke_name = "cluster_name" # Playground GKE cluster
name
+pg_datastore_namespace = "env" # Playground Datastore
namespace name
+tourofbeam_deployment_state_bucket = "tourofbeam-deployment-state-env" #
Bucket for Terraform state for deployment
+webhook_trigger_secret_id = "secret-cloudbuild-triggers-webhook" #
Secret ID for webhook trigger
+gh_pat_secret_id = "github_pat_playground_deployment" #
Secret ID for GitHub PAT
+tourofbeam_deploy_trigger_name = "TourOfBeam-Deploy-Update-env" #
Trigger name for deployment trigger
+tourofbeam_ci_trigger_name = "TourOfBeam-CI-env" # Trigger
name for CI trigger
+tourofbeam_cd_trigger_name = "TourOfBeam-CD-env" # Trigger
name for CD trigger
+tourofbeam_cb_private_bucket = "tourofbeam-cb-private-logs-env" # Bucket for
Cloud Build private logs
+tourofbeam_cb_public_bucket = "tourofbeam-cb-public-logs-env" # Bucket for
Cloud Build public logs
+web_app_id = "Tour-Of-Beam" = "tour-of-beam" # Web app ID
+
+If you plan to have only one environment, you can use simple resource names
like `tourofbeam-deployment-state` or `tourofbeam-cb-private-logs` instead of
`tourofbeam-deployment-state-env` or `tourofbeam-cb-private-logs-env`. But if
you plan to have multiple environments, it's advised to add environment name to
the resource name to avoid collisions with other environments.
+
+## 1. Set up the Google Cloud Build for your GCP project
+
+The `beam/learning/tour-of-beam/cloudbuild/01.setup` provisions dependencies
required to set up Cloud Build for Tour of Beam:
+- Required API services
+- Cloud Build triggers service accounts
+- IAM roles for Cloud Build service account
+
+#### To execute the module:
+
+1. Run commands:
+
+```console
+
+# Create a new authentication configuration for GCP Project with the created
user account
+gcloud init
+
+# Command imports new user account credentials into Application Default
Credentials
+gcloud auth application-default login
+
+# Navigate to 01.setup directory
+cd beam/learning/tour-of-beam/cloudbuild/01.setup
+
+# Run terraform commands
+terraform init -backend-config="bucket=\<tourofbeam-triggers-state-env\>
+terraform apply -var="project_id=$(gcloud config get-value project)"
-var-file="../common.tfvars"
+```
+
+## 2. Connect default (https://github.com/beamplayground/deploy-workaround)
GitHub repository with GCP Cloud Build
+
+Follow [Connect to a GitHub
repository](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github)
to connect GitHub repository with GCP Cloud Build.
+
+## 3. Set up the Google Cloud Build triggers
+
+The `beam/learning/tour-of-beam/cloudbuild/02.builders` provisions:
+- Cloud Build triggers to build and deploy Tour of Beam backend infrastructure
+
+#### To execute the module
+
+```
+# Navigate to beam/learning/tour-of-beam/cloudbuild/02.builders directory
+cd ../02.builders
+
+# Run terraform commands and provide required values
+terraform init -backend-config="bucket=\<tourofbeam-triggers-state-env\>"
+
+terraform apply -var="project_id=$(gcloud config get-value project)"
-var-file="../common.tfvars"
+
+```
+
+## 4. Run Cloud Build trigger to deploy Tour of Beam infrastructure, backend
and learning materials
+
+1. Navigate to [GCP Console Cloud Build
Triggers](https://console.cloud.google.com/cloud-build/triggers) page. Choose
the global region.
+2. `TourOfBeam-Deploy-Update-env`.
+3. Scroll down to `Source` - `Repository` to ensure that Default GitHub
repository is connected.
+ - Click on drop-down menu and press `CONNECT NEW REPOSITORY` in case it was
not automatically connected.
+4. Click `Save` and Run the trigger.
+
+## 5. Run Cloud Build trigger to deploy Tour of Beam infrastructure, backend
and learning materials
+
+1. Navigate to [GCP Console Cloud Build
Triggers](https://console.cloud.google.com/cloud-build/triggers) page. Choose
the global region.
+2. Open Trigger: `TourOfBeam-CD-env-manual`.
+3. Scroll down to `Source` - `Repository` to ensure that Default GitHub
repository is connected.
+ - Click on drop-down menu and press `CONNECT NEW REPOSITORY` in case it was
not automatically connected.
+4. Click `Save` and Run the trigger.
+
+## 6. Validate Tour of Beam backend infrastructure deployment
+1. Navigate to Cloud Functions service in GCP.
+2. Check if there are cloud functions with prefix of environment (e.g. prod,
test) in their names.
+3. Navigate to firebase web app url for newly created environment (e.g.
https://\<web_app_id\>-\<environment_name\>.web.app).
+4. Check if the web app is accessible and all learning materials and examples
are working.
\ No newline at end of file
diff --git
a/learning/tour-of-beam/cloudbuild/scripts/tob_deploy_infra_backend.sh
b/learning/tour-of-beam/cloudbuild/scripts/tob_deploy_infra_backend.sh
new file mode 100644
index 00000000000..d9714887412
--- /dev/null
+++ b/learning/tour-of-beam/cloudbuild/scripts/tob_deploy_infra_backend.sh
@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get -qq update
+
+apt-get -qq install -y wget unzip software-properties-common git curl
apt-transport-https ca-certificates gnupg jq lsb-release
+
+echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg]
https://packages.cloud.google.com/apt cloud-sdk main" | tee -a
/etc/apt/sources.list.d/google-cloud-sdk.list
+
+curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg
--dearmor -o /usr/share/keyrings/cloud.google.gpg
+
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o
/usr/share/keyrings/docker-archive-keyring.gpg
+
+echo "deb [arch=$(dpkg --print-architecture)
signed-by=/usr/share/keyrings/docker-archive-keyring.gpg]
https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" |tee
/etc/apt/sources.list.d/docker.list
+
+wget -nv
https://releases.hashicorp.com/terraform/1.4.2/terraform_1.4.2_linux_amd64.zip
+
+unzip terraform_1.4.2_linux_amd64.zip
+
+mv terraform /usr/local/bin/terraform
+
+wget -nv
https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_3.10.2-stable.tar.xz
+
+tar xf flutter_linux_3.10.2-stable.tar.xz
+
+git config --global --add safe.directory /usr/local/bin/flutter
+
+mv flutter /usr/local/bin/flutter
+
+export PATH="$PATH:/usr/local/bin/flutter/bin"
+
+flutter doctor
+
+wget -nv https://firebase.tools/bin/linux/latest
+
+mv latest /usr/local/bin/firebase
+
+chmod +x /usr/local/bin/firebase
+
+export PATH="$PATH:/usr/local/bin/"
+
+firebase --version
+
+apt-get -qq update
+
+apt-get -qq install -y google-cloud-sdk-gke-gcloud-auth-plugin
google-cloud-sdk openjdk-11-jdk kubectl docker-ce golang
+
+git clone --branch $BRANCH_NAME $REPO_NAME --single-branch
+
+gcloud auth configure-docker $PG_REGION-docker.pkg.dev
+
+gcloud container clusters get-credentials --region $PG_GKE_ZONE $PG_GKE_NAME
--project $PROJECT_ID
+
+cd beam/learning/tour-of-beam/terraform
+
+gcloud datastore indexes create ../backend/internal/storage/index.yaml
+
+terraform init -backend-config="bucket=${STATE_BUCKET}"
+
+terraform apply -auto-approve -var "pg_router_host=$(kubectl get svc -l
app=backend-router-grpc -o
jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}:{.items[0].spec.ports[0].port}')"
+
+cd ../../../
+
+export QUALIFIED_WEB_APP_ID=${WEB_APP_ID}-${TF_VAR_environment}
+
+./gradlew learning:tour-of-beam:terraform:InitFrontend -Pregion=$PG_REGION
-Pproject_id=$TF_VAR_project_id -Pproject_environment=$TF_VAR_environment
-Pdns-name=$DNS_NAME -Pwebapp_id=$QUALIFIED_WEB_APP_ID
+
+cd learning/tour-of-beam/backend
+
+go run ./cmd/ci_cd/ci_cd.go
+
diff --git a/learning/tour-of-beam/cloudbuild/scripts/tob_lm_cd.sh
b/learning/tour-of-beam/cloudbuild/scripts/tob_lm_cd.sh
new file mode 100644
index 00000000000..3cf35788472
--- /dev/null
+++ b/learning/tour-of-beam/cloudbuild/scripts/tob_lm_cd.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get -qq update
+
+apt-get -qq install -y wget unzip software-properties-common git curl
apt-transport-https ca-certificates gnupg jq lsb-release
+
+echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg]
https://packages.cloud.google.com/apt cloud-sdk main" | tee -a
/etc/apt/sources.list.d/google-cloud-sdk.list
+
+curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg
--dearmor -o /usr/share/keyrings/cloud.google.gpg
+
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o
/usr/share/keyrings/docker-archive-keyring.gpg
+
+echo "deb [arch=$(dpkg --print-architecture)
signed-by=/usr/share/keyrings/docker-archive-keyring.gpg]
https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" |tee
/etc/apt/sources.list.d/docker.list
+
+apt-get -qq update
+
+apt-get -qq install -y google-cloud-sdk golang
+
+git clone --branch master https://github.com/apache/beam.git --single-branch
+
+cd beam/learning/tour-of-beam/backend
+
+go run ./cmd/ci_cd/ci_cd.go
diff --git a/learning/tour-of-beam/terraform/README.md
b/learning/tour-of-beam/terraform/README.md
index 24b4d83e67e..1df6753c346 100644
--- a/learning/tour-of-beam/terraform/README.md
+++ b/learning/tour-of-beam/terraform/README.md
@@ -32,6 +32,9 @@ Before starting the deployment, ensure that you have the
following prerequisites
- Service Usage Admin
- Storage Admin
- Kubernetes Engine Cluster Viewer
+ - Firebase Admin
+ - Cloud Functions Admin
+
3. [Google Cloud Storage
bucket](https://cloud.google.com/storage/docs/creating-buckets) for saving
deployment state
@@ -90,8 +93,7 @@ terraform init -backend-config="bucket=`created_gcs_bucket`"
7. Run terraform apply to create Tour-Of-Beam backend infrastructure
```
-terraform plan -var "gcloud_init_account=$(gcloud config get-value
core/account)" \
--var "environment=prod" \
+terraform plan -var "environment=prod" \
-var "region=us-west1" \
-var "project_id=$(gcloud config get-value project)" \
-var "datastore_namespace=playground-datastore-namespace" \
@@ -99,8 +101,7 @@ terraform plan -var "gcloud_init_account=$(gcloud config
get-value core/account)
```
```
-terraform apply -var "gcloud_init_account=$(gcloud config get-value
core/account)" \
--var "environment=prod" \
+terraform apply -var "environment=prod" \
-var "region=us-west1" \
-var "project_id=$(gcloud config get-value project)" \
-var "datastore_namespace=playground-datastore-namespace" \
@@ -133,20 +134,6 @@ const cloudFunctionsBaseUrl = 'https://'
'$_cloudFunctionsProjectRegion-$_cloudFunctionsProjectId'
'.cloudfunctions.net/${environment}_';
-
-const String kAnalyticsUA = 'UA-73650088-2';
-const String kApiClientURL =
-'https://router.${dns_name}';
-const String kApiJavaClientURL =
-'https://java.${dns_name}';
-const String kApiGoClientURL =
-'https://go.${dns_name}';
-const String kApiPythonClientURL =
-'https://python.${dns_name}';
-const String kApiScioClientURL =
-'https://scio.${dns_name}';
-```
-
9. Create file .firebaserc under beam/learning/tour-of-beam/frontend
9.1. Navigate to beam/learning/tour-of-beam/frontend.
@@ -232,7 +219,7 @@ You will need to:
1) Remove "locationId" line.
2) Remove quotes (") from key of "key": "value" pair.
- 3) E.g. `projectId: "cloudbuild-384304"`
+3) E.g. `projectId: "cloudbuild-384304"`
4) In overall, redacted and ready to be inserted data should be as follows:
```
diff --git a/learning/tour-of-beam/terraform/build.gradle.kts
b/learning/tour-of-beam/terraform/build.gradle.kts
index fd24c822906..8c320dd242a 100644
--- a/learning/tour-of-beam/terraform/build.gradle.kts
+++ b/learning/tour-of-beam/terraform/build.gradle.kts
@@ -49,9 +49,8 @@ tasks.register<TerraformTask>("terraformRef") {
tasks.register<TerraformTask>("terraformApplyBackend") {
group = "backend-deploy"
- dependsOn("terraformInit")
-
- val pgRouterHost = if
(project.extensions.extraProperties.has("pg_router_host")) {
+
+ val pg_router_host = if
(project.extensions.extraProperties.has("pg_router_host")) {
project.extensions.extraProperties["pg_router_host"] as String
} else {
"unknown"
@@ -61,29 +60,29 @@ tasks.register<TerraformTask>("terraformApplyBackend") {
"-auto-approve",
"-lock=false",
"-parallelism=3",
- "-var=pg_router_host=$pgRouterHost",
- "-var=gcloud_init_account=$(gcloud config get-value core/account)",
+ "-var=pg_router_host=$pg_router_host",
"-var=project_id=$(gcloud config get-value project)",
"-var-file=./common.tfvars"
)
+
+ tasks.getByName("uploadLearningMaterials").mustRunAfter(this)
}
tasks.register<TerraformTask>("terraformDestroy") {
dependsOn("getRouterHost")
- val pgRouterHost = if
(project.extensions.extraProperties.has("pg_router_host")) {
+ val pg_router_host = if
(project.extensions.extraProperties.has("pg_router_host")) {
project.extensions.extraProperties["pg_router_host"] as String
} else {
"unknown"
}
args(
- "destroy",
- "-auto-approve",
- "-lock=false",
- "-var=pg_router_host=$pgRouterHost",
- "-var=gcloud_init_account=$(gcloud config get-value core/account)",
- "-var=project_id=$(gcloud config get-value project)",
- "-var-file=./common.tfvars"
+ "destroy",
+ "-auto-approve",
+ "-lock=false",
+ "-var=pg_router_host=$pg_router_host",
+ "-var=project_id=$(gcloud config get-value project)",
+ "-var-file=./common.tfvars"
)
}
@@ -147,6 +146,54 @@ tasks.register("firebaseProjectCreate") {
}
}
+tasks.register("firebaseHostingCreate") {
+ group = "frontend-deploy"
+ dependsOn("firebaseWebAppCreate")
+
+ doLast {
+ val projectId = project.property("project_id") as String
+ val webapp_id = project.property("webapp_id") as String
+ val listSitesResult = ByteArrayOutputStream()
+ exec {
+ executable("firebase")
+ args("hosting:sites:list", "--project", projectId)
+ workingDir("../frontend")
+ standardOutput = listSitesResult
+ }
+
+ println(listSitesResult)
+ val output = listSitesResult.toString()
+ val regex = "\\b$webapp_id\\b".toRegex()
+ if (regex.containsMatchIn(output)) {
+ println("Firebase is already added to project $projectId.")
+ } else {
+ exec {
+ executable("firebase")
+ args("hosting:sites:create", webapp_id)
+ workingDir("../frontend")
+ }.assertNormalExitValue()
+ println("Firebase hosting site has been added to project
$projectId.")
+ }
+
+ exec {
+ executable("firebase")
+ args("target:apply", "hosting", webapp_id , webapp_id)
+ workingDir("../frontend")
+
+ }.assertNormalExitValue()
+
+ val file = project.file("../frontend/firebase.json")
+ val content = file.readText()
+
+ val oldContent = """"public": "build/web","""
+ val newContent = """"public": "build/web",
+ "target": "$webapp_id","""
+ val updatedContent = content.replace(oldContent, newContent)
+
+ file.writeText(updatedContent)
+ }
+}
+
tasks.register("firebaseWebAppCreate") {
group = "frontend-deploy"
dependsOn("firebaseProjectCreate")
@@ -164,8 +211,9 @@ tasks.register("firebaseWebAppCreate") {
val output = result.toString()
if (output.contains(webappId)) {
println("Webapp id $webappId is already created on the project:
$projectId.")
- val regex = Regex("$webappId[│ ]+([\\w:]+)[│ ]+WEB[│ ]+")
+ val regex = Regex("""$webappId[\W]+([\d:a-zA-Z]+)[\W]+WEB""")
val firebaseAppId = regex.find(output)?.groupValues?.get(1)?.trim()
+ println("Firebase app ID for existing Firebase Web App:
$firebaseAppId")
project.extensions.extraProperties["firebaseAppId"] = firebaseAppId
} else {
val result2 = ByteArrayOutputStream()
@@ -185,7 +233,7 @@ tasks.register("firebaseWebAppCreate") {
// firebase apps:sdkconfig WEB AppId
tasks.register("getSdkConfigWebApp") {
group = "frontend-deploy"
- dependsOn("firebaseWebAppCreate")
+ dependsOn("firebaseHostingCreate")
doLast {
val firebaseAppId =
project.extensions.extraProperties["firebaseAppId"] as String
@@ -227,6 +275,7 @@ tasks.register("prepareFirebaseOptionsDart") {
}
tasks.register("flutterPubGetPG") {
+ dependsOn("prepareFirebaseOptionsDart")
doLast {
exec {
executable("flutter")
@@ -290,8 +339,9 @@ tasks.register("firebaseDeploy") {
doLast {
val projectId = project.property("project_id") as String
+ val webapp_id = project.property("webapp_id") as String
exec {
- commandLine("firebase", "deploy", "--project", projectId)
+ commandLine("firebase", "deploy", "--only", "hosting:$webapp_id",
"--project", projectId)
workingDir("../frontend")
}
}
@@ -317,18 +367,6 @@ const cloudFunctionsBaseUrl = 'https://'
'$region-$projectId'
'.cloudfunctions.net/${environment}_';
-
-const String kAnalyticsUA = 'UA-73650088-2';
-const String kApiClientURL =
-'https://router.${dnsName}';
-const String kApiJavaClientURL =
-'https://java.${dnsName}';
-const String kApiGoClientURL =
-'https://go.${dnsName}';
-const String kApiPythonClientURL =
-'https://python.${dnsName}';
-const String kApiScioClientURL =
-'https://scio.${dnsName}';
"""
)
}
@@ -397,6 +435,7 @@ tasks.register("InitFrontend") {
dependsOn("prepareFirebasercConfig")
dependsOn("firebaseProjectCreate")
dependsOn("firebaseWebAppCreate")
+ dependsOn("firebaseHostingCreate")
dependsOn("getSdkConfigWebApp")
dependsOn("prepareFirebaseOptionsDart")
dependsOn("flutterPubGetPG")
diff --git a/learning/tour-of-beam/terraform/cloud_functions/main.tf
b/learning/tour-of-beam/terraform/cloud_functions/main.tf
index e80225115db..e519c9378aa 100644
--- a/learning/tour-of-beam/terraform/cloud_functions/main.tf
+++ b/learning/tour-of-beam/terraform/cloud_functions/main.tf
@@ -22,7 +22,7 @@ resource "google_cloudfunctions_function" "cloud_function" {
runtime = "go116"
available_memory_mb = 128
project = var.project_id
- service_account_email = var.service_account_id
+ service_account_email = var.cf-service-account-id
source_archive_bucket = var.source_archive_bucket
source_archive_object = var.source_archive_object
region = var.region
diff --git a/learning/tour-of-beam/terraform/cloud_functions/variables.tf
b/learning/tour-of-beam/terraform/cloud_functions/variables.tf
index c7442359de6..258c52c9411 100644
--- a/learning/tour-of-beam/terraform/cloud_functions/variables.tf
+++ b/learning/tour-of-beam/terraform/cloud_functions/variables.tf
@@ -16,7 +16,7 @@
# under the License.
# Taken from output of SETUP module
-variable "service_account_id" {
+variable "cf-service-account-id" {
description = "The name of Service Account to run Cloud Function"
}
diff --git a/learning/tour-of-beam/terraform/functions_buckets/locals.tf
b/learning/tour-of-beam/terraform/functions_buckets/locals.tf
index ec2f63cb222..e909e43dd7b 100644
--- a/learning/tour-of-beam/terraform/functions_buckets/locals.tf
+++ b/learning/tour-of-beam/terraform/functions_buckets/locals.tf
@@ -15,21 +15,14 @@
# specific language governing permissions and limitations
# under the License.
-# Random string generator resource. To generate source code buckets name
-resource "random_string" "id" {
- length = 4
- upper = false
- special = false
-}
-
# Variable for prefix. Used in generated source code buckets name
variable "resource_name_prefix" {
type = string
description = "The resource name prefix applied to all resource naming for
the application"
- default = "tour-of-beam"
+ default = "tourofbeam"
}
# Local value to store generated GCS bucket name for source code (Cloud
Functions)
locals {
- cloudfunctions_bucket =
"${var.resource_name_prefix}-cfstorage-${random_string.id.result}"
+ cloudfunctions_bucket =
"${var.resource_name_prefix}-cfstorage-${var.environment}"
}
\ No newline at end of file
diff --git a/learning/tour-of-beam/terraform/functions_buckets/variables.tf
b/learning/tour-of-beam/terraform/functions_buckets/variables.tf
index f1cb931f2a2..b8ab4cf99cc 100644
--- a/learning/tour-of-beam/terraform/functions_buckets/variables.tf
+++ b/learning/tour-of-beam/terraform/functions_buckets/variables.tf
@@ -19,3 +19,7 @@
variable "region" {
description = "The GCP region where GCS bucket will be created (For Cloud
Functions source code)"
}
+
+variable "environment" {
+ description = "The name of the environment for deployment. Will create
directory where terraform config files will be stored"
+}
diff --git a/learning/tour-of-beam/terraform/main.tf
b/learning/tour-of-beam/terraform/main.tf
index ba122fc7acb..f37a4435a65 100644
--- a/learning/tour-of-beam/terraform/main.tf
+++ b/learning/tour-of-beam/terraform/main.tf
@@ -19,14 +19,15 @@
module "setup" {
source = "./setup"
project_id = var.project_id
- gcloud_init_account = var.gcloud_init_account
depends_on = [module.api_enable]
+ environment = var.environment
}
# GCS buckets to create buckets, objects, archive to store source code that
cloud functions will use
module "functions_buckets" {
source = "./functions_buckets"
region = var.region
+ environment = var.environment
depends_on = [module.setup, module.api_enable]
}
@@ -43,7 +44,7 @@ module "cloud_functions" {
pg_router_host = var.pg_router_host
environment = var.environment
datastore_namespace = var.datastore_namespace
- service_account_id = module.setup.service-account-email
+ cf-service-account-id = module.setup.cf-service-account-id
source_archive_bucket = module.functions_buckets.functions-bucket-name
source_archive_object = module.functions_buckets.function-bucket-object
depends_on = [module.functions_buckets, module.setup, module.api_enable]
diff --git a/learning/tour-of-beam/terraform/setup/iam.tf
b/learning/tour-of-beam/terraform/setup/iam.tf
index 245872364fd..0b0eaf28abb 100644
--- a/learning/tour-of-beam/terraform/setup/iam.tf
+++ b/learning/tour-of-beam/terraform/setup/iam.tf
@@ -18,7 +18,7 @@
# Service account for GCP Cloud Functions
resource "google_service_account" "cloud_function_sa" {
account_id = local.cloudfunctions_service_account
- display_name = "Service Account to run Cloud Functions"
+ display_name = "Tour of Beam CF Service Account-${var.environment}"
}
# IAM roles for Cloud Functions service account
@@ -32,13 +32,3 @@ resource "google_project_iam_member"
"terraform_service_account_roles" {
member = "serviceAccount:${google_service_account.cloud_function_sa.email}"
project = var.project_id
}
-
-# IAM roles to be granted for user account that will be running terraform
scripts
-resource "google_project_iam_member" "gcloud_user_required_roles" {
- for_each = toset([
- "roles/cloudfunctions.admin", "roles/firebase.admin"
- ])
- role = each.key
- member = "user:${var.gcloud_init_account}"
- project = var.project_id
-}
diff --git a/learning/tour-of-beam/terraform/setup/locals.tf
b/learning/tour-of-beam/terraform/setup/locals.tf
index 9817b8f2821..6ebf0e359e6 100644
--- a/learning/tour-of-beam/terraform/setup/locals.tf
+++ b/learning/tour-of-beam/terraform/setup/locals.tf
@@ -17,18 +17,13 @@
# Local value to store generated Cloud Functions' Service account name
-resource "random_string" "id" {
- length = 4
- upper = false
- special = false
-}
variable "resource_name_prefix" {
type = string
description = "The resource name prefix applied to all resource naming for
the application"
- default = "tour-of-beam"
+ default = "tourofbeam"
}
locals {
- cloudfunctions_service_account =
"${var.resource_name_prefix}-cf-sa-${random_string.id.result}"
+ cloudfunctions_service_account =
"${var.resource_name_prefix}-cf-sa-${var.environment}"
}
\ No newline at end of file
diff --git a/learning/tour-of-beam/terraform/setup/output.tf
b/learning/tour-of-beam/terraform/setup/output.tf
index bd069f1ed8d..4ae6894e130 100644
--- a/learning/tour-of-beam/terraform/setup/output.tf
+++ b/learning/tour-of-beam/terraform/setup/output.tf
@@ -16,6 +16,6 @@
# under the License.
# Output used to assign service account to cloud functions
-output "service-account-email" {
+output "cf-service-account-id" {
value = google_service_account.cloud_function_sa.email
}
\ No newline at end of file
diff --git a/learning/tour-of-beam/terraform/setup/variables.tf
b/learning/tour-of-beam/terraform/setup/variables.tf
index dcb6f54808b..f5c231a2851 100644
--- a/learning/tour-of-beam/terraform/setup/variables.tf
+++ b/learning/tour-of-beam/terraform/setup/variables.tf
@@ -21,7 +21,7 @@ variable "project_id" {
description = "The ID of the Google Cloud project within which resources are
provisioned"
}
-# This variable is generated by command (gcloud config get-value core/account)
in Kotlin Gradle script
-variable "gcloud_init_account" {
- description = "User Account ID logged in with gcloud init command (e.g.
[email protected])"
+variable "environment" {
+ description = "The name of the environment for deployment. Will create
directory where terraform config files will be stored"
}
+
diff --git a/learning/tour-of-beam/terraform/variables.tf
b/learning/tour-of-beam/terraform/variables.tf
index 0a045514dd5..760d6614587 100644
--- a/learning/tour-of-beam/terraform/variables.tf
+++ b/learning/tour-of-beam/terraform/variables.tf
@@ -16,7 +16,6 @@
# under the License.
# GCP Project ID
-# Provided as a result of gcloud command
variable "project_id" {
description = "The ID of the Google Cloud project within which resources are
provisioned"
}
@@ -26,12 +25,6 @@ variable "region" {
description = "The region of the Google Cloud project within which resources
are provisioned"
}
-# User account that will be deploying Tour of Beam infrastructure
-# Provided as a result of gcloud command
-variable "gcloud_init_account" {
- description = "User Account ID logged in with gcloud init command (e.g.
[email protected])"
-}
-
# Existing Playground router hostname:port details
# Provided as a result of kubectl command
variable "pg_router_host" {
@@ -46,4 +39,4 @@ variable "environment" {
variable "datastore_namespace" {
description = "The name of datastore namespace"
-}
\ No newline at end of file
+}
diff --git
a/playground/frontend/playground_components/lib/src/constants/backend_urls.dart
b/playground/frontend/playground_components/lib/src/constants/backend_urls.dart
index 670dfea2dad..20c2d6348d9 100644
---
a/playground/frontend/playground_components/lib/src/constants/backend_urls.dart
+++
b/playground/frontend/playground_components/lib/src/constants/backend_urls.dart
@@ -19,7 +19,7 @@
// ignore_for_file: prefer_interpolation_to_compose_strings
/// The template for production backend URL.
-const defaultBackendUrlTemplate = 'https://{node}.play-dev.beam.apache.org';
+const defaultBackendUrlTemplate = 'https://{node}.play.beam.apache.org';
/// The URLs for local backend development.
const backendUrlOverrides = <String, String>{
