rmetzger commented on a change in pull request #10976: [FLINK-13978][build system] Add experimental support for building on Azure Pipelines URL: https://github.com/apache/flink/pull/10976#discussion_r378393653
########## File path: tools/azure_controller.sh ########## @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +echo $M2_HOME +echo $PATH +echo $MAVEN_OPTS + +mvn -version +echo "Commit: $(git rev-parse HEAD)" + +# Set up a custom Maven settings file, configuring an Google-hosted maven central +# mirror. +cat << EOF > /tmp/az_settings.xml +<settings> + <mirrors> + <mirror> + <id>google-maven-central</id> + <name>GCS Maven Central mirror</name> + <url>https://maven-central.storage-download.googleapis.com/repos/central/data/</url> + <mirrorOf>central</mirrorOf> + </mirror> + </mirrors> +</settings> +EOF + + +HERE="`dirname \"$0\"`" # relative +HERE="`( cd \"$HERE\" && pwd )`" # absolutized and normalized +if [ -z "$HERE" ] ; then + # error; for some reason, the path is not accessible + # to the script (e.g. permissions re-evaled after suid) + exit 1 # fail +fi + +source "${HERE}/travis/stage.sh" +source "${HERE}/travis/shade.sh" + +print_system_info() { + echo "CPU information" + lscpu + + echo "Memory information" + cat /proc/meminfo + + echo "Disk information" + df -hH + + echo "Running build as" + whoami +} + +print_system_info + + +STAGE=$1 +echo "Current stage: \"$STAGE\"" + +EXIT_CODE=0 + +#adding -Dmaven.wagon.http.pool=false (see https://developercommunity.visualstudio.com/content/problem/851041/microsoft-hosted-agents-run-into-maven-central-tim.html) +# --settings /tmp/az_settings.xml +MVN="mvn clean install $MAVEN_OPTS -nsu -Dflink.convergence.phase=install -Pcheck-convergence -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.wagon.http.pool=false -Dmaven.javadoc.skip=true -B -U -DskipTests $PROFILE" + +# Run actual compile&test steps +if [ $STAGE == "$STAGE_COMPILE" ]; then + # run mvn clean install: + $MVN + EXIT_CODE=$? + + if [ $EXIT_CODE == 0 ]; then + echo "\n\n==============================================================================\n" + echo "Checking scala suffixes\n" + echo "==============================================================================\n" + + ./tools/verify_scala_suffixes.sh "${PROFILE}" + EXIT_CODE=$? + else + echo "\n==============================================================================\n" + echo "Previous build failure detected, skipping scala-suffixes check.\n" + echo "==============================================================================\n" + fi + + if [ $EXIT_CODE == 0 ]; then + check_shaded_artifacts + EXIT_CODE=$(($EXIT_CODE+$?)) + check_shaded_artifacts_s3_fs hadoop + EXIT_CODE=$(($EXIT_CODE+$?)) + check_shaded_artifacts_s3_fs presto + EXIT_CODE=$(($EXIT_CODE+$?)) + check_shaded_artifacts_connector_elasticsearch 2 + EXIT_CODE=$(($EXIT_CODE+$?)) + check_shaded_artifacts_connector_elasticsearch 5 + EXIT_CODE=$(($EXIT_CODE+$?)) + check_shaded_artifacts_connector_elasticsearch 6 + EXIT_CODE=$(($EXIT_CODE+$?)) + else + echo "==============================================================================" + echo "Previous build failure detected, skipping shaded dependency check." + echo "==============================================================================" + fi + + if [ $EXIT_CODE == 0 ]; then + echo "Creating cache build directory $CACHE_FLINK_DIR" + + cp -r . "$CACHE_FLINK_DIR" + + function minimizeCachedFiles() { + # reduces the size of the cached directory to speed up + # the packing&upload / download&unpacking process + # by removing files not required for subsequent stages + + # jars are re-built in subsequent stages, so no need to cache them (cannot be avoided) + find "$CACHE_FLINK_DIR" -maxdepth 8 -type f -name '*.jar' \ + ! -path "$CACHE_FLINK_DIR/flink-formats/flink-csv/target/flink-csv*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-formats/flink-json/target/flink-json*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-formats/flink-avro/target/flink-avro*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-runtime/target/flink-runtime*tests.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-streaming-java/target/flink-streaming-java*tests.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/lib/flink-dist*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/lib/flink-table_*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/lib/flink-table-blink*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/opt/flink-python*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-connectors/flink-connector-elasticsearch-base/target/flink-*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-connectors/flink-connector-kafka-base/target/flink-*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-table/flink-table-planner/target/flink-table-planner*tests.jar" | xargs rm -rf + + # .git directory + # not deleting this can cause build stability issues + # merging the cached version sometimes fails + rm -rf "$CACHE_FLINK_DIR/.git" + + # AZ Pipelines has a problem with links. + rm "$CACHE_FLINK_DIR/build-target" + } + + echo "Minimizing cache" + minimizeCachedFiles + else + echo "==============================================================================" + echo "Previous build failure detected, skipping cache setup." + echo "==============================================================================" + fi +elif [ $STAGE != "$STAGE_CLEANUP" ]; then + if ! [ -e $CACHE_FLINK_DIR ]; then + echo "Cached flink dir $CACHE_FLINK_DIR does not exist. Exiting build." + exit 1 + fi + # merged compiled flink into local clone + # this prevents the cache from being re-uploaded + echo "Merging cache" + cp -RT "$CACHE_FLINK_DIR" "." + + echo "Adjusting timestamps" + # adjust timestamps to prevent recompilation + find . -type f -name '*.java' | xargs touch + find . -type f -name '*.scala' | xargs touch + # wait a bit for better odds of different timestamps + sleep 5 + find . -type f -name '*.class' | xargs touch + find . -type f -name '*.timestamp' | xargs touch + + if [ $STAGE == $STAGE_PYTHON ]; then + echo "==============================================================================" + echo "Python stage found. Re-compiling (this is required on Azure for the python tests to pass)" Review comment: I hoped to have found a solution, but it still didn't work. I'm happy to consider this a quick fix and revisit this problem once the PR is merged. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
