This is an automated email from the ASF dual-hosted git repository. gaborgsomogyi pushed a commit to branch ci_utils in repository https://gitbox.apache.org/repos/asf/flink-connector-shared-utils.git
The following commit(s) were added to refs/heads/ci_utils by this push: new 7691962 [FLINK-33556][CI] Test infrastructure for externalized python code 7691962 is described below commit 7691962b8031536a2baa4d39252a38198ba91dc5 Author: pvary <peter.vary.apa...@gmail.com> AuthorDate: Thu Dec 7 10:33:52 2023 +0100 [FLINK-33556][CI] Test infrastructure for externalized python code --- .github/workflows/python_ci.yml | 86 ++++ python/README.md | 6 + python/build-wheels.sh | 52 +++ python/glibc_version_fix.h | 17 + python/install_command.sh | 31 ++ python/lint-python.sh | 876 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 1068 insertions(+) diff --git a/.github/workflows/python_ci.yml b/.github/workflows/python_ci.yml new file mode 100644 index 0000000..89e10d9 --- /dev/null +++ b/.github/workflows/python_ci.yml @@ -0,0 +1,86 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +on: + workflow_call: + inputs: + flink_version: + description: "Flink version to test against." + required: true + type: string + timeout_global: + description: "The timeout in minutes for the entire workflow." + required: false + type: number + default: 80 + timeout_test: + description: "The timeout in minutes for the test compile&step." + required: false + type: number + default: 50 + connector_branch: + description: "Branch that need to be checked out" + required: false + type: string + +jobs: + python_test: + runs-on: ubuntu-latest + timeout-minutes: ${{ inputs.timeout_global }} + env: + MVN_COMMON_OPTIONS: -U -B --no-transfer-progress -Dflink.version=${{ inputs.flink_version }} -DskipTests + MVN_CONNECTION_OPTIONS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 + MVN_BUILD_OUTPUT_FILE: "/tmp/mvn_build_output.out" + steps: + - run: echo "Running CI pipeline for JDK version 8" + + - name: Check out repository code + uses: actions/checkout@v3 + with: + ref: "${{ inputs.connector_branch }}" + + - name: Set JDK + uses: actions/setup-java@v3 + with: + java-version: 8 + distribution: 'temurin' + cache: 'maven' + + - name: Set Maven 3.8.6 + uses: stCarolas/setup-maven@v4.5 + with: + maven-version: 3.8.6 + + - name: Compile + timeout-minutes: ${{ inputs.timeout_test }} + run: | + set -o pipefail + + mvn clean install ${MVN_COMMON_OPTIONS} \ + ${{ env.MVN_CONNECTION_OPTIONS }} \ + -Dlog4j.configurationFile=file://$(pwd)/tools/ci/log4j.properties \ + | tee ${{ env.MVN_BUILD_OUTPUT_FILE }} + + - name: Run Python test + timeout-minutes: ${{ inputs.timeout_test }} + run: | + set -o pipefail + + cd flink-python + chmod a+x dev/* + ./dev/lint-python.sh -e mypy,sphinx | tee ${{ env.MVN_BUILD_OUTPUT_FILE }} diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..0dc8217 --- /dev/null +++ b/python/README.md @@ -0,0 +1,6 @@ +This directory contains commonly used scripts for testing and creating python packages for connectors. + +The original version of the files are based on +https://github.com/apache/flink/tree/release-1.17.2/flink-python/dev. + +Created FLINK-33762 to make these scripts versioned to allow backward incompatible changes in the future. diff --git a/python/build-wheels.sh b/python/build-wheels.sh new file mode 100755 index 0000000..7f18a91 --- /dev/null +++ b/python/build-wheels.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e -x + +## 1. install python env +dev/lint-python.sh -s py_env + +PY_ENV_DIR=`pwd`/dev/.conda/envs +py_env=("3.7" "3.8" "3.9" "3.10") +## 2. install dependency +for ((i=0;i<${#py_env[@]};i++)) do + ${PY_ENV_DIR}/${py_env[i]}/bin/pip install -r dev/dev-requirements.txt +done + +## 3. build wheels +for ((i=0;i<${#py_env[@]};i++)) do + if [[ "$(uname)" != "Darwin" ]]; then + # force the linker to use the older glibc version in Linux + export CFLAGS="-I. -include dev/glibc_version_fix.h" + fi + ${PY_ENV_DIR}/${py_env[i]}/bin/python setup.py bdist_wheel +done + +## 4. convert linux_x86_64 wheel to manylinux1 wheel in Linux +if [[ "$(uname)" != "Darwin" ]]; then + source `pwd`/dev/.conda/bin/activate + # 4.1 install patchelf + conda install -c conda-forge patchelf=0.11 -y + # 4.2 install auditwheel + pip install auditwheel==3.2.0 + # 4.3 convert Linux wheel + for wheel_file in dist/*.whl; do + auditwheel repair ${wheel_file} -w dist + rm -f ${wheel_file} + done + source deactivate +fi +## see the result +ls -al dist/ diff --git a/python/glibc_version_fix.h b/python/glibc_version_fix.h new file mode 100755 index 0000000..f260492 --- /dev/null +++ b/python/glibc_version_fix.h @@ -0,0 +1,17 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +__asm__(".symver memcpy,memcpy@GLIBC_2.2.5"); diff --git a/python/install_command.sh b/python/install_command.sh new file mode 100755 index 0000000..f914200 --- /dev/null +++ b/python/install_command.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +retry_times=3 +install_command="python -m pip install $@" +${install_command} +status=$? +while [[ ${status} -ne 0 ]] && [[ ${retry_times} -gt 0 ]]; do + retry_times=$((retry_times-1)) + # sleep 3 seconds and then reinstall. + sleep 3 + ${install_command} + status=$? +done + +exit ${status} diff --git a/python/lint-python.sh b/python/lint-python.sh new file mode 100755 index 0000000..1c0452b --- /dev/null +++ b/python/lint-python.sh @@ -0,0 +1,876 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# lint-python.sh +# This script will prepare a virtual environment for many kinds of checks, such as tox check, flake8 check. +# +# You can refer to the README.MD in ${flink-python} to learn how easy to run the script. +# + +# Download some software, such as miniconda.sh +function download() { + local DOWNLOAD_STATUS= + if hash "wget" 2>/dev/null; then + # because of the difference of all versions of wget, so we turn of the option --show-progress + wget "$1" -O "$2" -q -T20 -t3 + DOWNLOAD_STATUS="$?" + else + curl "$1" -o "$2" --progress-bar --connect-timeout 20 --retry 3 + DOWNLOAD_STATUS="$?" + fi + if [ $DOWNLOAD_STATUS -ne 0 ]; then + echo "Download failed.You can try again" + exit $DOWNLOAD_STATUS + fi +} + +# Printing infos both in log and console +function print_function() { + local STAGE_LENGTH=48 + local left_edge_len= + local right_edge_len= + local str + case "$1" in + "STAGE") + left_edge_len=$(((STAGE_LENGTH-${#2})/2)) + right_edge_len=$((STAGE_LENGTH-${#2}-left_edge_len)) + str="$(seq -s "=" $left_edge_len | tr -d "[:digit:]")""$2""$(seq -s "=" $right_edge_len | tr -d "[:digit:]")" + ;; + "STEP") + str="$2" + ;; + *) + str="seq -s "=" $STAGE_LENGTH | tr -d "[:digit:]"" + ;; + esac + echo $str | tee -a $LOG_FILE +} + +function regexp_match() { + if echo $1 | grep -e $2 &>/dev/null; then + echo true + else + echo false + fi +} + +# decide whether a array contains a specified element. +function contains_element() { + arr=($1) + if echo "${arr[@]}" | grep -w "$2" &>/dev/null; then + echo true + else + echo false + fi +} + +# Checkpoint the stage:step for convenient to re-exec the script with +# skipping those success steps. +# The format is "${Stage}:${Step}". e.g. Install:4 +function checkpoint_stage() { + if [ ! -d `dirname $STAGE_FILE` ]; then + mkdir -p `dirname $STAGE_FILE` + fi + echo "$1:$2">"$STAGE_FILE" +} + +# Restore the stage:step +function restore_stage() { + if [ -f "$STAGE_FILE" ]; then + local lines=$(awk '{print NR}' $STAGE_FILE) + if [ $lines -eq 1 ]; then + local first_field=$(cat $STAGE_FILE | cut -d ":" -f 1) + local second_field=$(cat $STAGE_FILE | cut -d ":" -f 2) + check_valid_stage $first_field $second_field + if [ $? -eq 0 ]; then + STAGE=$first_field + STEP=$second_field + return + fi + fi + fi + STAGE="install" + STEP=0 +} + +# Decide whether the stage:step is valid. +function check_valid_stage() { + case $1 in + "install") + if [ $2 -le $STAGE_INSTALL_STEPS ] && [ $2 -ge 0 ]; then + return 0 + fi + ;; + *) + ;; + esac + return 1 +} + +function parse_component_args() { + local REAL_COMPONENTS=() + for component in ${INSTALLATION_COMPONENTS[@]}; do + # because all other components depends on conda, the install of conda is + # required component. + if [[ "$component" == "basic" ]] || [[ "$component" == "miniconda" ]]; then + continue + fi + if [[ "$component" == "all" ]]; then + component="environment" + fi + if [[ `contains_element "${SUPPORTED_INSTALLATION_COMPONENTS[*]}" "${component}"` = true ]]; then + REAL_COMPONENTS+=(${component}) + else + echo "unknown install component ${component}, currently we only support installing basic,py_env,tox,flake8,sphinx,mypy,all." + exit 1 + fi + done + if [[ `contains_element "${REAL_COMPONENTS[*]}" "environment"` = false ]]; then + SUPPORTED_INSTALLATION_COMPONENTS=(${REAL_COMPONENTS[@]}) + fi +} + +# For convenient to index something binded to OS. +# Now, the script only make a distinction between 'Mac' and 'Non-Mac'. +function get_os_index() { + if [ $1 == "Darwin" ]; then + return 0 + else + return 1 + fi +} + +# Considering the file size of miniconda.sh, +# "wget" is better than "curl" in the weak network environment. +function install_wget() { + if [ $1 == "Darwin" ]; then + hash "brew" 2>/dev/null + if [ $? -ne 0 ]; then + $((/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)") 2>&1 >/dev/null) + if [ $? -ne 0 ]; then + echo "Failed to install brew" + exit 1 + fi + fi + + hash "wget" 2>/dev/null + if [ $? -ne 0 ]; then + brew install wget 2>&1 >/dev/null + if [ $? -ne 0 ]; then + echo "Failed to install wget" + exit 1 + fi + fi + fi +} + +# The script choose miniconda as our package management tool. +# The script use miniconda to create all kinds of python versions and +# some packages including checks such as tox and flake8. + +function install_miniconda() { + OS_TO_CONDA_URL=("https://repo.continuum.io/miniconda/Miniconda3-4.7.10-MacOSX-x86_64.sh" \ + "https://repo.continuum.io/miniconda/Miniconda3-4.7.10-Linux-x86_64.sh") + if [ ! -f "$CONDA_INSTALL" ]; then + print_function "STEP" "download miniconda..." + download ${OS_TO_CONDA_URL[$1]} $CONDA_INSTALL_SH + chmod +x $CONDA_INSTALL_SH + if [ $? -ne 0 ]; then + echo "Please manually chmod +x $CONDA_INSTALL_SH" + exit 1 + fi + if [ -d "$CURRENT_DIR/.conda" ]; then + rm -rf "$CURRENT_DIR/.conda" + if [ $? -ne 0 ]; then + echo "Please manually rm -rf $CURRENT_DIR/.conda directory.\ + Then retry to exec the script." + exit 1 + fi + fi + print_function "STEP" "download miniconda... [SUCCESS]" + fi + + if [ ! -d "$CURRENT_DIR/.conda" ]; then + print_function "STEP" "installing conda..." + $CONDA_INSTALL_SH -b -p $CURRENT_DIR/.conda 2>&1 >/dev/null + + # orjson depend on pip >= 20.3 + print_function "STEP" "upgrade pip..." + $CURRENT_DIR/.conda/bin/python -m pip install --upgrade pip 2>&1 >/dev/null + print_function "STEP" "upgrade pip... [SUCCESS]" + + if [ $? -ne 0 ]; then + echo "install miniconda failed" + exit $CONDA_INSTALL_STATUS + fi + print_function "STEP" "install conda ... [SUCCESS]" + fi +} + +# Install some kinds of py env. +function install_py_env() { + if [[ ${BUILD_REASON} = 'IndividualCI' ]]; then + py_env=("3.10") + else + py_env=("3.7" "3.8" "3.9" "3.10") + fi + for ((i=0;i<${#py_env[@]};i++)) do + if [ -d "$CURRENT_DIR/.conda/envs/${py_env[i]}" ]; then + rm -rf "$CURRENT_DIR/.conda/envs/${py_env[i]}" + if [ $? -ne 0 ]; then + echo "rm -rf $CURRENT_DIR/.conda/envs/${py_env[i]} failed, please \ + rm -rf $CURRENT_DIR/.conda/envs/${py_env[i]} manually.\ + Then retry to exec the script." + exit 1 + fi + fi + print_function "STEP" "installing python${py_env[i]}..." + max_retry_times=3 + retry_times=0 + install_command="$CONDA_PATH create --name ${py_env[i]} -y -q python=${py_env[i]}" + ${install_command} 2>&1 >/dev/null + status=$? + while [[ ${status} -ne 0 ]] && [[ ${retry_times} -lt ${max_retry_times} ]]; do + retry_times=$((retry_times+1)) + # sleep 3 seconds and then reinstall. + sleep 3 + echo "conda install ${py_env[i]} retrying ${retry_times}/${max_retry_times}" + ${install_command} 2>&1 >/dev/null + status=$? + done + if [[ ${status} -ne 0 ]]; then + echo "conda install ${py_env[i]} failed after retrying ${max_retry_times} times.\ + You can retry to execute the script again." + exit 1 + fi + print_function "STEP" "install python${py_env[i]}... [SUCCESS]" + done +} + +# Install tox. +# In some situations,you need to run the script with "sudo". e.g. sudo ./lint-python.sh +function install_tox() { + source $CONDA_HOME/bin/activate + if [ -f "$TOX_PATH" ]; then + $PIP_PATH uninstall tox -y -q 2>&1 >/dev/null + if [ $? -ne 0 ]; then + echo "pip uninstall tox failed \ + please try to exec the script again.\ + if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f" + exit 1 + fi + fi + + # tox 3.14.0 depends on both 0.19 and 0.23 of importlib_metadata at the same time and + # conda will try to install both these two versions and it will cause problems occasionally. + # Using pip as the package manager could avoid this problem. + $CURRENT_DIR/install_command.sh -q virtualenv==20.10.0 tox==3.14.0 2>&1 >/dev/null + if [ $? -ne 0 ]; then + echo "pip install tox failed \ + please try to exec the script again.\ + if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f" + exit 1 + fi + conda deactivate +} + +# Install flake8. +# In some situations,you need to run the script with "sudo". e.g. sudo ./lint-python.sh +function install_flake8() { + source $CONDA_HOME/bin/activate + if [ -f "$FLAKE8_PATH" ]; then + $PIP_PATH uninstall flake8 -y -q 2>&1 >/dev/null + if [ $? -ne 0 ]; then + echo "pip uninstall flake8 failed \ + please try to exec the script again.\ + if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f" + exit 1 + fi + fi + + $CURRENT_DIR/install_command.sh -q flake8==4.0.1 2>&1 >/dev/null + if [ $? -ne 0 ]; then + echo "pip install flake8 failed \ + please try to exec the script again.\ + if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f" + exit 1 + fi + conda deactivate +} + +# Install sphinx. +# In some situations,you need to run the script with "sudo". e.g. sudo ./lint-python.sh +function install_sphinx() { + source $CONDA_HOME/bin/activate + if [ -f "$SPHINX_PATH" ]; then + $PIP_PATH uninstall Sphinx -y -q 2>&1 >/dev/null + if [ $? -ne 0 ]; then + echo "pip uninstall sphinx failed \ + please try to exec the script again.\ + if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f" + exit 1 + fi + fi + + $CURRENT_DIR/install_command.sh -q Sphinx==4.5.0 importlib-metadata==4.4.0 Docutils==0.17.1 pydata_sphinx_theme==0.11.0 sphinx_mdinclude==0.5.3 "Jinja2<3.1.0" 2>&1 >/dev/null + if [ $? -ne 0 ]; then + echo "pip install sphinx failed \ + please try to exec the script again.\ + if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f" + exit 1 + fi + conda deactivate +} + + +# Install mypy. +# In some situations, you need to run the script with "sudo". e.g. sudo ./lint-python.sh +function install_mypy() { + source ${CONDA_HOME}/bin/activate + if [[ -f "$MYPY_PATH" ]]; then + ${PIP_PATH} uninstall mypy -y -q 2>&1 >/dev/null + if [[ $? -ne 0 ]]; then + echo "pip uninstall mypy failed \ + please try to exec the script again.\ + if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f" + exit 1 + fi + fi + ${CURRENT_DIR}/install_command.sh -q mypy==0.790 2>&1 >/dev/null + if [[ $? -ne 0 ]]; then + echo "pip install mypy failed \ + please try to exec the script again.\ + if failed many times, you can try to exec in the form of sudo ./lint-python.sh -f" + exit 1 + fi + conda deactivate +} + +function need_install_component() { + if [[ `contains_element "${SUPPORTED_INSTALLATION_COMPONENTS[*]}" "$1"` = true ]]; then + echo true + else + echo false + fi +} + + +# In this function, the script will prepare all kinds of python environments and checks. +function install_environment() { + + print_function "STAGE" "installing environment" + + local sys_os=`uname -s` + #get the index of the SUPPORT_OS array for convinient to intall tool. + get_os_index $sys_os + local os_index=$? + + # step-1 install wget + # the file size of the miniconda.sh is too big to use "wget" tool to download instead + # of the "curl" in the weak network environment. + if [ $STEP -lt 1 ]; then + print_function "STEP" "installing wget..." + install_wget ${SUPPORT_OS[$os_index]} + STEP=1 + checkpoint_stage $STAGE $STEP + print_function "STEP" "install wget... [SUCCESS]" + fi + + # step-2 install miniconda + if [ $STEP -lt 2 ]; then + print_function "STEP" "installing miniconda..." + create_dir $CURRENT_DIR/download + install_miniconda $os_index + STEP=2 + checkpoint_stage $STAGE $STEP + print_function "STEP" "install miniconda... [SUCCESS]" + fi + + # step-3 install python environment which includes + # 3.7 3.8 3.9 3.10 + if [ $STEP -lt 3 ] && [ `need_install_component "py_env"` = true ]; then + print_function "STEP" "installing python environment..." + install_py_env + STEP=3 + checkpoint_stage $STAGE $STEP + print_function "STEP" "install python environment... [SUCCESS]" + fi + + # step-4 install tox + if [ $STEP -lt 4 ] && [ `need_install_component "tox"` = true ]; then + print_function "STEP" "installing tox..." + install_tox + STEP=4 + checkpoint_stage $STAGE $STEP + print_function "STEP" "install tox... [SUCCESS]" + fi + + # step-5 install flake8 + if [ $STEP -lt 5 ] && [ `need_install_component "flake8"` = true ]; then + print_function "STEP" "installing flake8..." + install_flake8 + STEP=5 + checkpoint_stage $STAGE $STEP + print_function "STEP" "install flake8... [SUCCESS]" + fi + + # step-6 install sphinx + if [ $STEP -lt 6 ] && [ `need_install_component "sphinx"` = true ]; then + print_function "STEP" "installing sphinx..." + install_sphinx + STEP=6 + checkpoint_stage $STAGE $STEP + print_function "STEP" "install sphinx... [SUCCESS]" + fi + + # step-7 install mypy + if [[ ${STEP} -lt 7 ]] && [[ `need_install_component "mypy"` = true ]]; then + print_function "STEP" "installing mypy..." + install_mypy + STEP=7 + checkpoint_stage ${STAGE} ${STEP} + print_function "STEP" "install mypy... [SUCCESS]" + fi + + print_function "STAGE" "install environment... [SUCCESS]" +} + +# create dir if needed +function create_dir() { + if [ ! -d $1 ]; then + mkdir -p $1 + if [ $? -ne 0 ]; then + echo "mkdir -p $1 failed. you can mkdir manually or exec the script with \ + the command: sudo ./lint-python.sh" + exit 1 + fi + fi +} + +# Set created py-env in $PATH for tox's creating virtual env +function activate () { + if [ ! -d $CURRENT_DIR/.conda/envs ]; then + echo "For some unknown reasons, missing the directory $CURRENT_DIR/.conda/envs,\ + you should exec the script with the option: -f" + exit 1 + fi + + for py_dir in $CURRENT_DIR/.conda/envs/* + do + PATH=$py_dir/bin:$PATH + done + export PATH 2>/dev/null + if [ $? -ne 0 ]; then + echo "For some unknown reasons, the py package is not complete,\ + you should exec the script with the option: -f" + exit 1 + fi +} + +# Reset the $PATH +function deactivate() { + # reset old environment variables + # ! [ -z ${VAR+_} ] returns true if VAR is declared at all + if ! [ -z "${_OLD_PATH+_}" ] ; then + PATH="$_OLD_PATH" + export PATH + unset _OLD_PATH + fi +} + +# Collect checks +function collect_checks() { + if [ ! -z "$EXCLUDE_CHECKS" ] && [ ! -z "$INCLUDE_CHECKS" ]; then + echo "You can't use option -s and -e simultaneously." + exit 1 + fi + if [ ! -z "$EXCLUDE_CHECKS" ]; then + for (( i = 0; i < ${#EXCLUDE_CHECKS[@]}; i++)); do + if [[ `contains_element "${SUPPORT_CHECKS[*]}" "${EXCLUDE_CHECKS[i]}_check"` = true ]]; then + SUPPORT_CHECKS=("${SUPPORT_CHECKS[@]/${EXCLUDE_CHECKS[i]}_check}") + else + echo "the check ${EXCLUDE_CHECKS[i]} is invalid." + exit 1 + fi + done + fi + if [ ! -z "$INCLUDE_CHECKS" ]; then + REAL_SUPPORT_CHECKS=() + for (( i = 0; i < ${#INCLUDE_CHECKS[@]}; i++)); do + if [[ `contains_element "${SUPPORT_CHECKS[*]}" "${INCLUDE_CHECKS[i]}_check"` = true ]]; then + REAL_SUPPORT_CHECKS+=("${INCLUDE_CHECKS[i]}_check") + else + echo "the check ${INCLUDE_CHECKS[i]} is invalid." + exit 1 + fi + done + SUPPORT_CHECKS=(${REAL_SUPPORT_CHECKS[@]}) + fi +} + +# If the check stage is needed +function include_stage() { + if [[ `contains_element "${SUPPORT_CHECKS[*]}" "$1"` = true ]]; then + return 0 + else + return 1 + fi +} + +# get all supported checks functions +function get_all_supported_checks() { + _OLD_IFS=$IFS + IFS=$'\n' + SUPPORT_CHECKS=() + for fun in $(declare -F); do + if [[ `regexp_match "$fun" "_check$"` = true ]]; then + SUPPORT_CHECKS+=("${fun:11}") + fi + done + IFS=$_OLD_IFS +} + +# get all supported install components functions +function get_all_supported_install_components() { + _OLD_IFS=$IFS + IFS=$'\n' + for fun in $(declare -F); do + if [[ `regexp_match "${fun:11}" "^install_"` = true ]]; then + SUPPORTED_INSTALLATION_COMPONENTS+=("${fun:19}") + fi + done + IFS=$_OLD_IFS + # we don't need to expose "install_wget" to user. + local DELETE_COMPONENTS=("wget") + local REAL_COMPONENTS=() + for component in ${SUPPORTED_INSTALLATION_COMPONENTS[@]}; do + if [[ `contains_element "${DELETE_COMPONENTS[*]}" "${component}"` = false ]]; then + REAL_COMPONENTS+=("${component}") + fi + done + SUPPORTED_INSTALLATION_COMPONENTS=(${REAL_COMPONENTS[@]}) +} + +# exec all selected check stages +function check_stage() { + print_function "STAGE" "checks starting" + for fun in ${SUPPORT_CHECKS[@]}; do + $fun + done + echo "All the checks are finished, the detailed information can be found in: $LOG_FILE" +} + + +###############################################################All Checks Definitions############################################################### +######################### +# This part defines all check functions such as tox_check and flake8_check +# We make a rule that all check functions are suffixed with _ check. e.g. tox_check, flake8_chek +######################### +# Tox check +function tox_check() { + LATEST_PYTHON="py310" + print_function "STAGE" "tox checks" + # Set created py-env in $PATH for tox's creating virtual env + activate + # Ensure the permission of the scripts set correctly + chmod +x $FLINK_PYTHON_DIR/../build-target/bin/* + chmod +x $FLINK_PYTHON_DIR/dev/* + + if [[ ${BUILD_REASON} = 'IndividualCI' ]]; then + # Only run test in latest python version triggered by a Git push + $TOX_PATH -vv -c $FLINK_PYTHON_DIR/tox.ini -e ${LATEST_PYTHON} --recreate 2>&1 | tee -a $LOG_FILE + else + $TOX_PATH -vv -c $FLINK_PYTHON_DIR/tox.ini --recreate 2>&1 | tee -a $LOG_FILE + fi + + TOX_RESULT=$((grep -c "congratulations :)" "$LOG_FILE") 2>&1) + if [ $TOX_RESULT -eq '0' ]; then + print_function "STAGE" "tox checks... [FAILED]" + else + print_function "STAGE" "tox checks... [SUCCESS]" + fi + # Reset the $PATH + deactivate + # If check failed, stop the running script. + if [ $TOX_RESULT -eq '0' ]; then + exit 1 + fi +} + +# Flake8 check +function flake8_check() { + local PYTHON_SOURCE="$(find . \( -path ./dev -o -path ./.tox \) -prune -o -type f -name "*.py" -print )" + + print_function "STAGE" "flake8 checks" + if [ ! -f "$FLAKE8_PATH" ]; then + echo "For some unknown reasons, the flake8 package is not complete,\ + you should exec the script with the parameter: -f" + fi + + if [[ ! "$PYTHON_SOURCE" ]]; then + echo "No python files found! Something is wrong exiting." + exit 1; + fi + + # the return value of a pipeline is the status of the last command to exit + # with a non-zero status or zero if no command exited with a non-zero status + set -o pipefail + ($FLAKE8_PATH --config=tox.ini $PYTHON_SOURCE) 2>&1 | tee -a $LOG_FILE + + PYCODESTYLE_STATUS=$? + if [ $PYCODESTYLE_STATUS -ne 0 ]; then + print_function "STAGE" "flake8 checks... [FAILED]" + # Stop the running script. + exit 1; + else + print_function "STAGE" "flake8 checks... [SUCCESS]" + fi +} + +# Sphinx check +function sphinx_check() { + export SPHINXBUILD=$SPHINX_PATH + # cd to $FLINK_PYTHON_DIR + pushd "$FLINK_PYTHON_DIR"/docs &> /dev/null + make clean + + # the return value of a pipeline is the status of the last command to exit + # with a non-zero status or zero if no command exited with a non-zero status + set -o pipefail + (SPHINXOPTS="-a -W" make html) 2>&1 | tee -a $LOG_FILE + + SPHINXBUILD_STATUS=$? + if [ $SPHINXBUILD_STATUS -ne 0 ]; then + print_function "STAGE" "sphinx checks... [FAILED]" + # Stop the running script. + exit 1; + else + print_function "STAGE" "sphinx checks... [SUCCESS]" + fi + popd +} + +# mypy check +function mypy_check() { + print_function "STAGE" "mypy checks" + + # the return value of a pipeline is the status of the last command to exit + # with a non-zero status or zero if no command exited with a non-zero status + set -o pipefail + (${MYPY_PATH} --config-file tox.ini) 2>&1 | tee -a ${LOG_FILE} + TYPE_HINT_CHECK_STATUS=$? + if [ ${TYPE_HINT_CHECK_STATUS} -ne 0 ]; then + print_function "STAGE" "mypy checks... [FAILED]" + # Stop the running script. + exit 1; + else + print_function "STAGE" "mypy checks... [SUCCESS]" + fi +} +###############################################################All Checks Definitions############################################################### + +# CURRENT_DIR is "flink/flink-python/dev/" +CURRENT_DIR="$(cd "$( dirname "$0" )" && pwd)" + +# FLINK_PYTHON_DIR is "flink/flink-python" +FLINK_PYTHON_DIR=$(dirname "$CURRENT_DIR") + +# conda home path +if [ -z "${FLINK_CONDA_HOME+x}" ]; then + CONDA_HOME="$CURRENT_DIR/.conda" + ENV_HOME="$CONDA_HOME" +else + CONDA_HOME=$FLINK_CONDA_HOME + ENV_HOME="${CONDA_PREFIX-$CONDA_HOME}" +fi + +# conda path +CONDA_PATH=$CONDA_HOME/bin/conda + +# pip path +PIP_PATH=$ENV_HOME/bin/pip + +# tox path +TOX_PATH=$ENV_HOME/bin/tox + +# flake8 path +FLAKE8_PATH=$ENV_HOME/bin/flake8 + +# sphinx path +SPHINX_PATH=$ENV_HOME/bin/sphinx-build + +# mypy path +MYPY_PATH=$ENV_HOME/bin/mypy + +_OLD_PATH="$PATH" + +SUPPORT_OS=("Darwin" "Linux") + +# the file stores the success step in installing progress. +STAGE_FILE=$CURRENT_DIR/.stage.txt + +# the dir includes all kinds of py env installed. +VIRTUAL_ENV=$CONDA_HOME/envs + +LOG_DIR=$CURRENT_DIR/log + +if [ "$FLINK_IDENT_STRING" == "" ]; then + FLINK_IDENT_STRING="$USER" +fi +if [ "$HOSTNAME" == "" ]; then + HOSTNAME="$HOST" +fi + +# the log file stores the checking result. +LOG_FILE=$LOG_DIR/flink-$FLINK_IDENT_STRING-python-$HOSTNAME.log +create_dir $LOG_DIR + +# clean LOG_FILE content +echo >$LOG_FILE + +# miniconda script +CONDA_INSTALL_SH=$CURRENT_DIR/download/miniconda.sh + +# stage "install" includes the num of steps. +STAGE_INSTALL_STEPS=7 + +# whether force to restart the script. +FORCE_START=0 + +SUPPORT_CHECKS=() + +# search all supported check functions and put them into SUPPORT_CHECKS array +get_all_supported_checks + +EXCLUDE_CHECKS="" + +INCLUDE_CHECKS="" + +SUPPORTED_INSTALLATION_COMPONENTS=() + +# search all supported install functions and put them into SUPPORTED_INSTALLATION_COMPONENTS array +get_all_supported_install_components + +INSTALLATION_COMPONENTS=() + +# whether remove the installed python environment. +CLEAN_UP_FLAG=0 + +# parse_opts +USAGE=" +usage: $0 [options] +-h print this help message and exit +-f force to exec from the progress of installing environment +-s [basic,py_env,tox,flake8,sphinx,mypy,all] + install environment with specified components which split by comma(,) + note: + This option is used to install environment components and will skip all subsequent checks, + so do not use this option with -e,-i simultaneously. +-e [tox,flake8,sphinx,mypy] + exclude checks which split by comma(,) +-i [tox,flake8,sphinx,mypy] + include checks which split by comma(,) +-l list all checks supported. +Examples: + ./lint-python -s basic => install environment with basic components. + ./lint-python -s py_env => install environment with python env(3.7,3.8,3.9,3.10). + ./lint-python -s all => install environment with all components such as python env,tox,flake8,sphinx,mypy etc. + ./lint-python -s tox,flake8 => install environment with tox,flake8. + ./lint-python -s tox -f => reinstall environment with tox. + ./lint-python -e tox,flake8 => exclude checks tox,flake8. + ./lint-python -i flake8 => include checks flake8. + ./lint-python => exec all checks. + ./lint-python -f => reinstall environment with all components and exec all checks. + ./lint-python -l => list all checks supported. + ./lint-python -r => clean up python environment. +" +while getopts "hfs:i:e:lr" arg; do + case "$arg" in + h) + printf "%s\\n" "$USAGE" + exit 2 + ;; + f) + FORCE_START=1 + ;; + s) + INSTALLATION_COMPONENTS=($(echo $OPTARG | tr ',' ' ' )) + ;; + e) + EXCLUDE_CHECKS=($(echo $OPTARG | tr ',' ' ' )) + ;; + i) + INCLUDE_CHECKS=($(echo $OPTARG | tr ',' ' ' )) + ;; + l) + printf "current supported checks includes:\n" + for fun in ${SUPPORT_CHECKS[@]}; do + echo ${fun%%_check*} + done + exit 2 + ;; + r) + printf "clean up python environment:\n" + CLEAN_UP_FLAG=1 + ;; + ?) + printf "ERROR: did not recognize option '%s', please try -h\\n" "$1" + exit 1 + ;; + esac +done + +# decides whether to skip check stage +skip_checks=0 + +if [[ ${CLEAN_UP_FLAG} -eq 1 ]]; then + printf "clean up python environment" + rm -rf ${CONDA_HOME} + rm -rf ${STAGE_FILE} + rm -rf ${FLINK_PYTHON_DIR}/.tox + skip_checks=1 +fi + +if [ ! -z "$INSTALLATION_COMPONENTS" ]; then + parse_component_args + skip_checks=1 +fi + +# collect checks according to the options +collect_checks + +# If exec the script with the param: -f, all progress will be re-run +if [ $FORCE_START -eq 1 ]; then + STAGE="install" + STEP=0 + checkpoint_stage $STAGE $STEP +else + restore_stage +fi + +# install environment +if [[ ${CLEAN_UP_FLAG} -eq 0 ]]; then + install_environment +fi + +pushd "$FLINK_PYTHON_DIR" &> /dev/null +# exec all selected checks +if [ $skip_checks -eq 0 ]; then + check_stage +fi