TSultanov commented on code in PR #25117: URL: https://github.com/apache/beam/pull/25117#discussion_r1178854864
########## playground/infrastructure/cloudbuild/playground_ci_examples.sh: ########## @@ -0,0 +1,246 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script validates examples against runners (Java, Python, Go) built as local containers from a current branch +# +# Command line arguments: +# LOG_PATH - full path lo a log file. Output is also logged to stdout (Default: /dev/null) +# BEAM_ROOT_DIR - Path (full) to cloned repo root. Used by ci_cd.py (Default: /workspace/beam) +# PROJECT_ID - GCP Project ID. Used by ci_cd.py (Default: test) +# BEAM_CONCURRENCY - Number of examples to run in parallel. Used by ci_cd.py (Default: 4) +# BEAM_VERSION - version of BEAM SDK to build containers +# SUBDIRS - array of paths (relative to beam repo root) to search changed examples (Default: ./learning/katas ./examples ./sdks) +# COMMIT - Git commit hash to build containers from (Default: HEAD) +# DIFF_BASE - Git branch to compare with $COMMIT to look for changed examples (Default: origin/master) +# SDKS - Array of SDKS to validate (Default: java python go) +# ORIGIN - examples origin (Default: PG_EXAMPLES) +# ALLOWLIST - List of paths (relative to the repo root) not in examples (SUBDIRS) that cause reb + +for ARGUMENT in "$@" +do + KEY=$(echo $ARGUMENT | cut -f1 -d=) + + KEY_LENGTH=${#KEY} + VALUE="${ARGUMENT:$KEY_LENGTH+1}" + + export "$KEY"="$VALUE" +done + +export LOG_PATH=${LOG_PATH-"/dev/null"} +export BEAM_ROOT_DIR=${BEAM_ROOT_DIR-"/workspace/beam"} +export PROJECT_ID=${PROJECT_ID-"test"} +export BEAM_VERSION=${BEAM_VERSION-"2.44.0"} +export SUBDIRS=${SUBDIRS-"./learning/katas ./examples ./sdks"} +export SDKS=${SDKS-"java python go"} +export COMMIT=${COMMIT-"HEAD"} +export DIFF_BASE=${DIFF_BASE-"origin/master"} +export BEAM_CONCURRENCY=${BEAM_CONCURRENCY-"4"} +export ORIGIN=${ORIGIN-"PG_EXAMPLES"} +export ALLOWLIST=${ALLOWLIST-"playground/infrastructure playground/backend"} + +function LogOutput () +{ + echo "$(date --utc '+%D %T') $1" >> $LOG_PATH + # CILOG keyword to simplify search over the global log + echo "CILOG $(date --utc '+%D %T') $1" +} + + +LogOutput "Input variables: + LOG_PATH=$LOG_PATH + BEAM_ROOT_DIR=$BEAM_ROOT_DIR + PROJECT_ID=$PROJECT_ID + BEAM_VERSION=$BEAM_VERSION + SUBDIRS=$SUBDIRS + SDKS=$SDKS + COMMIT=$COMMIT + BEAM_CONCURRENCY=$BEAM_CONCURRENCY + ORIGIN=$ORIGIN + ALLOWLIST=$ALLOWLIST" + +# Assigning constant values +# Script starts in a clean environment in Cloud Build. Set minimal required environment variables +if [ -z "$PATH" ]; then + export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin/:/usr/bin:/sbin:/bin" +fi +if [ -z "$HOME" ]; then + export HOME="/builder/home" +fi +export STEP=CI +export SDK_CONFIG="$BEAM_ROOT_DIR/playground/sdks.yaml" +export BEAM_EXAMPLE_CATEGORIES="$BEAM_ROOT_DIR/playground/categories.yaml" +export GRADLE_VERSION=7.5.1 +export GO_VERSION=1.18 + +LogOutput "Installing python java8 and dependencies" +apt-get update > /dev/null +apt update > /dev/null +export DEBIAN_FRONTEND=noninteractive + +LogOutput "Installing Python environment" +apt-get install -y apt-transport-https ca-certificates software-properties-common curl unzip apt-utils > /dev/null +add-apt-repository -y ppa:deadsnakes/ppa > /dev/null && apt update > /dev/null Review Comment: Again, there is no reason to use non-system version of python -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
