This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch release-1.13
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.13 by this push:
new fdabee0 [FLINK-22987][ci] Fix scala suffix check
fdabee0 is described below
commit fdabee0d3720f6d1690a7df7fb9329d266b49add
Author: Chesnay Schepler <[email protected]>
AuthorDate: Mon Jun 21 09:01:37 2021 +0200
[FLINK-22987][ci] Fix scala suffix check
* [FLINK-22987][ci] Fix scala suffix check
* temporarily disable avro registry
* Add basic precondition that at least 1 clean&infected module must be found
* Invert exit handling
---
tools/ci/compile.sh | 2 +-
.../tools/ci/suffixcheck/ScalaSuffixChecker.java | 254 +++++++++++++++++++++
tools/ci/verify_scala_suffixes.sh | 137 ++---------
3 files changed, 273 insertions(+), 120 deletions(-)
diff --git a/tools/ci/compile.sh b/tools/ci/compile.sh
index 14e03fe..e442bb8 100755
--- a/tools/ci/compile.sh
+++ b/tools/ci/compile.sh
@@ -94,7 +94,7 @@ cd ..
echo "============ Checking scala suffixes ============"
-${CI_DIR}/verify_scala_suffixes.sh "${PROFILE}" || exit $?
+${CI_DIR}/verify_scala_suffixes.sh "$CI_DIR" "$(pwd)" || exit $?
echo "============ Checking shaded dependencies ============"
diff --git
a/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/suffixcheck/ScalaSuffixChecker.java
b/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/suffixcheck/ScalaSuffixChecker.java
new file mode 100644
index 0000000..65d1e40
--- /dev/null
+++
b/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/suffixcheck/ScalaSuffixChecker.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.tools.ci.suffixcheck;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/** Utility for checking the presence/absence of scala-suffixes. */
+public class ScalaSuffixChecker {
+ private static final Logger LOG =
LoggerFactory.getLogger(ScalaSuffixChecker.class);
+
+ // [INFO] --- maven-dependency-plugin:3.1.1:tree (default-cli) @
flink-annotations ---
+ private static final Pattern moduleNamePattern =
+ Pattern.compile(".* --- maven-dependency-plugin.* @ (.*) ---.*");
+
+ // [INFO] +- junit:junit:jar:4.13.2:test
+ // [INFO] | \- org.hamcrest:hamcrest-core:jar:1.3:test
+ // [INFO] \- org.apache.logging.log4j:log4j-1.2-api:jar:2.14.1:test
+ private static final Pattern blockPattern = Pattern.compile(".*
[+|\\\\].*");
+
+ // [INFO] +- org.scala-lang:scala-reflect:jar:2.11.12:test
+ private static final Pattern scalaSuffixPattern =
Pattern.compile("_2.1[0-9]");
+
+ public static void main(String[] args) throws IOException {
+ if (args.length < 2) {
+ System.out.println("Usage: ScalaSuffixChecker
<pathMavenBuildOutput> <pathFlinkRoot>");
+ System.exit(1);
+ }
+
+ final Path mavenOutputPath = Paths.get(args[0]);
+ final Path flinkRootPath = Paths.get(args[1]);
+
+ final ParseResult parseResult = parseMavenOutput(mavenOutputPath);
+ if (parseResult.getCleanModules().isEmpty()) {
+ LOG.error("Parsing found 0 scala-free modules; the parsing is
likely broken.");
+ System.exit(1);
+ }
+ if (parseResult.getInfectedModules().isEmpty()) {
+ LOG.error("Parsing found 0 scala-dependent modules; the parsing is
likely broken.");
+ System.exit(1);
+ }
+
+ final Collection<String> violations = checkScalaSuffixes(parseResult,
flinkRootPath);
+
+ if (!violations.isEmpty()) {
+ LOG.error(
+ "Violations found:{}",
+ violations.stream().collect(Collectors.joining("\n\t",
"\n\t", "")));
+ System.exit(1);
+ }
+ }
+
+ private static ParseResult parseMavenOutput(final Path path) throws
IOException {
+ final Set<String> cleanModules = new HashSet<>();
+ final Set<String> infectedModules = new HashSet<>();
+
+ try (final BufferedReader bufferedReader =
+ Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
+
+ String line;
+ while ((line = bufferedReader.readLine()) != null) {
+ final Matcher matcher = moduleNamePattern.matcher(line);
+ if (matcher.matches()) {
+ final String moduleName =
stripScalaSuffix(matcher.group(1));
+ LOG.trace("Parsing module '{}'.", moduleName);
+
+ // skip: [INFO]
org.apache.flink:flink-annotations:jar:1.14-SNAPSHOT
+ bufferedReader.readLine();
+
+ boolean infected = false;
+ line = bufferedReader.readLine();
+ while (blockPattern.matcher(line).matches()) {
+ final boolean dependsOnScala = dependsOnScala(line);
+ final boolean isTestDependency =
line.endsWith(":test");
+ LOG.trace("\tline:{}", line);
+ LOG.trace("\t\tdepends-on-scala:{}", dependsOnScala);
+ LOG.trace("\t\tis-test-dependency:{}",
isTestDependency);
+ if (dependsOnScala && !isTestDependency) {
+ LOG.trace("\t\tOutbreak detected at {}!",
moduleName);
+ infected = true;
+ }
+
+ line = bufferedReader.readLine();
+ }
+
+ if (infected) {
+ infectedModules.add(moduleName);
+ } else {
+ cleanModules.add(moduleName);
+ }
+ }
+ }
+ }
+
+ return new ParseResult(cleanModules, infectedModules);
+ }
+
+ private static String stripScalaSuffix(final String moduleName) {
+ final int i = moduleName.indexOf("_2.");
+ return i > 0 ? moduleName.substring(0, i) : moduleName;
+ }
+
+ private static boolean dependsOnScala(final String line) {
+ return line.contains("org.scala-lang") ||
scalaSuffixPattern.matcher(line).find();
+ }
+
+ private static Collection<String> checkScalaSuffixes(
+ final ParseResult parseResult, Path flinkRootPath) throws
IOException {
+ final Collection<String> violations = new ArrayList<>();
+
+ // exclude e2e modules and flink-docs for convenience as they
+ // a) are not deployed during a release
+ // b) exist only for dev purposes
+ // c) no-one should depend on them
+ final Collection<String> excludedModules = new ArrayList<>();
+ excludedModules.add("flink-docs");
+ excludedModules.addAll(getEndToEndTestModules(flinkRootPath));
+ // temporary; see FLINK-23001
+ excludedModules.add("flink-avro-glue-schema-registry");
+
+ for (String excludedModule : excludedModules) {
+ parseResult.getCleanModules().remove(excludedModule);
+ parseResult.getInfectedModules().remove(excludedModule);
+ }
+
+ violations.addAll(checkCleanModules(parseResult.getCleanModules(),
flinkRootPath));
+
violations.addAll(checkInfectedModules(parseResult.getInfectedModules(),
flinkRootPath));
+
+ return violations;
+ }
+
+ private static Collection<String> getEndToEndTestModules(Path
flinkRootPath)
+ throws IOException {
+ try (Stream<Path> pathStream =
+ Files.walk(flinkRootPath.resolve("flink-end-to-end-tests"),
5)) {
+ return pathStream
+ .filter(path ->
path.getFileName().toString().equals("pom.xml"))
+ .map(path -> path.getParent().getFileName().toString())
+ .collect(Collectors.toList());
+ }
+ }
+
+ private static Collection<String> checkCleanModules(
+ Collection<String> modules, Path flinkRootPath) throws IOException
{
+ return checkModules(
+ modules,
+ flinkRootPath,
+ "_${scala.binary.version}",
+ "Scala-free module '%s' is referenced with scala suffix in
'%s'.");
+ }
+
+ private static Collection<String> checkInfectedModules(
+ Collection<String> modules, Path flinkRootPath) throws IOException
{
+ return checkModules(
+ modules,
+ flinkRootPath,
+ "",
+ "Scala-dependent module '%s' is referenced without scala
suffix in '%s'.");
+ }
+
+ private static Collection<String> checkModules(
+ Collection<String> modules,
+ Path flinkRootPath,
+ String moduleSuffix,
+ String violationTemplate)
+ throws IOException {
+
+ final ArrayList<String> sortedModules = new ArrayList<>(modules);
+ sortedModules.sort(String::compareTo);
+
+ final Collection<String> violations = new ArrayList<>();
+ for (String module : sortedModules) {
+ int numPreviousViolations = violations.size();
+ try (Stream<Path> pathStream = Files.walk(flinkRootPath, 3)) {
+ final List<Path> pomFiles =
+ pathStream
+ .filter(path ->
path.getFileName().toString().equals("pom.xml"))
+ .collect(Collectors.toList());
+
+ for (Path pomFile : pomFiles) {
+ try (Stream<String> lines = Files.lines(pomFile,
StandardCharsets.UTF_8)) {
+ final boolean existsCleanReference =
+ lines.anyMatch(
+ line ->
+ line.contains(
+ module + moduleSuffix
+ "</artifactId>"));
+
+ if (existsCleanReference) {
+ violations.add(
+ String.format(
+ violationTemplate,
+ module,
+
flinkRootPath.relativize(pomFile)));
+ }
+ }
+ }
+ }
+ if (numPreviousViolations == violations.size()) {
+ LOG.info("OK {}", module);
+ }
+ }
+ return violations;
+ }
+
+ private static class ParseResult {
+
+ private final Set<String> cleanModules;
+ private final Set<String> infectedModules;
+
+ private ParseResult(Set<String> cleanModules, Set<String>
infectedModules) {
+ this.cleanModules = cleanModules;
+ this.infectedModules = infectedModules;
+ }
+
+ public Set<String> getCleanModules() {
+ return cleanModules;
+ }
+
+ public Set<String> getInfectedModules() {
+ return infectedModules;
+ }
+ }
+}
diff --git a/tools/ci/verify_scala_suffixes.sh
b/tools/ci/verify_scala_suffixes.sh
index 8fd6589..c6189d3 100755
--- a/tools/ci/verify_scala_suffixes.sh
+++ b/tools/ci/verify_scala_suffixes.sh
@@ -32,138 +32,37 @@
# correct suffixes, i.e. modules with Scala dependency should carry a suffix
# but modules without Scala should be suffix-free.
#
-# The script may be run from the /tools directory or the Flink root. Prior to
-# executing the script it is advised to run 'mvn clean install -DskipTests' to
-# build and install the latest version of Flink.
+# Prior to executing the script it is advised to run 'mvn clean install
-DskipTests'
+# to build and install the latest version of Flink.
#
# The script uses 'mvn dependency:tree -Dincludes=org.scala-lang' to list Scala
# dependent modules.
-#
-# Some example output:
-#
-### Example of a dependency-free module:
-# [INFO]
------------------------------------------------------------------------
-# [INFO] Building flink-java 1.0-SNAPSHOT
-# [INFO]
------------------------------------------------------------------------
-# [INFO]
-# [INFO] --- maven-dependency-plugin:2.8:tree (default-cli) @ flink-java ---
-# [INFO]
-
-### Example of a Scala dependent module:
-# [INFO]
------------------------------------------------------------------------
-# [INFO] Building flink-storm 1.0-SNAPSHOT
-# [INFO]
------------------------------------------------------------------------
-# [INFO]
-# [INFO] --- maven-dependency-plugin:2.8:tree (default-cli) @ flink-storm ---
-# [INFO] org.apache.flink:flink-storm:jar:1.0-SNAPSHOT
-# [INFO] \- org.apache.flink:flink-streaming-java:jar:1.0-SNAPSHOT:compile
-# [INFO] \- org.apache.flink:flink-runtime:jar:1.0-SNAPSHOT:compile
-# [INFO] \- org.scala-lang:scala-library:jar:2.10.4:compile
-# [INFO]
-
-MAVEN_ARGUMENTS=${1:-""}
-
-if [[ `basename $PWD` == "tools" ]] ; then
- cd ..
-fi
+CI_DIR=$1
+FLINK_ROOT=$2
echo "--- Flink Scala Dependency Analyzer ---"
-
-# Parser for mvn:dependency output
-
-BEGIN="[INFO] Building flink"
-SEPARATOR="[INFO]
------------------------------------------------------------------------"
-END=$SEPARATOR
-
-reached_block=0
-in_block=0
-block_name=""
-block_infected=0
-
-# exclude e2e modules and flink-docs for convenience as they
-# a) are not deployed during a release
-# b) exist only for dev purposes
-# c) no-one should depend on them
-e2e_modules=$(find flink-end-to-end-tests -mindepth 2 -maxdepth 5 -name
'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',')
-excluded_modules=\!${e2e_modules//,/,\!},!flink-docs,!flink-end-to-end-tests
-
echo "Analyzing modules for Scala dependencies using 'mvn dependency:tree'."
echo "If you haven't built the project, please do so first by running \"mvn
clean install -DskipTests\""
-infected=""
-clean=""
-
-while read line; do
- if [[ $line == *"$BEGIN"* ]]; then
- reached_block=1
- # Maven module name
- block_name=`[[ "$line" =~ .*(flink-?[-a-zA-Z0-9.]*).* ]] && echo
${BASH_REMATCH[1]}`
- elif [[ $line == *"$SEPARATOR"* ]] && [[ $reached_block -eq 1 ]]; then
- reached_block=0
- in_block=1
- elif [[ $line == *"$END"* ]] && [[ $in_block -eq 1 ]]; then
- if [[ $block_infected -eq 0 ]]; then
- clean="$block_name $clean"
- fi
- in_block=0
- reached_block=0
- block_name=""
- block_infected=0
- elif [[ $in_block -eq 1 ]]; then
- echo $line | grep -E "org.scala-lang|- [^:]+:[^:]+_2\.1[0-9]" | grep
--invert-match "org.scala-lang.*:.*:.*:test" | grep --invert-match
"[^:]*:[^:]*_2\.1[0-9]:.*:.*:test" >/dev/null
- if [ $? -eq 0 ]; then
- #echo $block_name
- infected="$block_name $infected"
- block_infected=1
- fi
- fi
-done < <(run_mvn dependency:tree -Dincludes=org.scala-lang,:*_2.1*:: -pl
${excluded_modules} ${MAVEN_ARGUMENTS} | tee /dev/tty)
-
-
-# deduplicate and sort
-clean=`echo $clean | xargs -n1 | sort -u`
+source "${CI_DIR}/maven-utils.sh"
-# deduplicate and sort
-infected=`echo $infected | xargs -n1 | sort -u`
+cd "$FLINK_ROOT" || exit
+dependency_plugin_output=${CI_DIR}/dep.txt
-# #
-# ### Check whether the suffixes are correct
-# #
+run_mvn dependency:tree -Dincludes=org.scala-lang,:*_2.1*:: ${MAVEN_ARGUMENTS}
>> "${dependency_plugin_output}"
-RED='\e[0;31m'
-GREEN='\e[0;32m'
-NC='\033[0m'
+cd "${CI_DIR}/java-ci-tools/" || exit
-exit_code=0
+run_mvn exec:java
-Dexec.mainClass=org.apache.flink.tools.ci.suffixcheck.ScalaSuffixChecker
-Dexec.args=\""${dependency_plugin_output}" "${FLINK_ROOT}"\"
+EXIT_CODE=$?
-echo
-echo "Checking Scala-free modules:"
-
-for module in $clean; do
- out=`find . -maxdepth 3 -name 'pom.xml' -not -path '*target*' -exec grep
"${module}_\\${scala.binary.version}</artifactId>" "{}" \;`
- if [[ "$out" == "" ]]; then
- printf "$GREEN OK $NC $module\n"
- else
- printf "$RED NOT OK $NC $module\n"
- echo "$out"
- exit_code=1
- fi
-done
-
-echo
-echo "Checking Scala-dependent modules:"
-
-for module in $infected; do
- out=`find . -maxdepth 3 -name 'pom.xml' -not -path '*target*' -exec grep
"${module}</artifactId>" "{}" \;`
- if [[ "$out" == "" ]]; then
- printf "$GREEN OK $NC $module\n"
- else
- printf "$RED NOT OK $NC $module\n"
- echo "$out"
- exit_code=1
- fi
-done
+if [ $EXIT_CODE == 0 ]; then
+ exit 0
+fi
-exit $exit_code
+echo
"=============================================================================="
+echo "Suffix Check failed. See previous output for details."
+echo
"=============================================================================="
+exit 1
