This is an automated email from the ASF dual-hosted git repository. elek pushed a commit to branch ozone-0.4.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 97897b6ab747e973fd05feac70e4c89bce3daa07 Author: Márton Elek <[email protected]> AuthorDate: Sat Jun 29 01:59:44 2019 +0200 HDDS-1735. Create separate unit and integration test executor dev-support script (cherry picked from commit 62a057b8d647730498ea9a04d57f18b4520d09cf) --- hadoop-ozone/dev-support/checks/acceptance.sh | 6 +++++- hadoop-ozone/dev-support/checks/author.sh | 14 ++++++++++---- hadoop-ozone/dev-support/checks/build.sh | 5 ++++- hadoop-ozone/dev-support/checks/checkstyle.sh | 10 ++++++++-- hadoop-ozone/dev-support/checks/findbugs.sh | 12 +++++++----- .../dev-support/checks/{unit.sh => integration.sh} | 12 ++++++++---- hadoop-ozone/dev-support/checks/isolation.sh | 7 +++++-- hadoop-ozone/dev-support/checks/rat.sh | 17 +++++++++++++---- .../dev-support/checks/{acceptance.sh => shellcheck.sh} | 15 ++++++++++++--- hadoop-ozone/dev-support/checks/unit.sh | 8 ++++---- 10 files changed, 76 insertions(+), 30 deletions(-) diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/acceptance.sh index 8de920f..4a50e08 100755 --- a/hadoop-ozone/dev-support/checks/acceptance.sh +++ b/hadoop-ozone/dev-support/checks/acceptance.sh @@ -14,6 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../../.." || exit 1 + export HADOOP_VERSION=3 -"$DIR/../../../hadoop-ozone/dist/target/ozone-*-SNAPSHOT/compose/test-all.sh" +OZONE_VERSION=$(grep "<ozone.version>" "$DIR/../../pom.xml" | sed 's/<[^>]*>//g'| sed 's/^[ \t]*//') +cd "$DIR/../../dist/target/ozone-$OZONE_VERSION/compose" || exit 1 +./test-all.sh exit $? diff --git a/hadoop-ozone/dev-support/checks/author.sh b/hadoop-ozone/dev-support/checks/author.sh index 43caa70..d5a469c 100755 --- a/hadoop-ozone/dev-support/checks/author.sh +++ b/hadoop-ozone/dev-support/checks/author.sh @@ -13,10 +13,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -mkdir -p ./target -grep -r --include="*.java" "@author" . -if [ $? -gt 0 ]; then +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../../.." || exit 1 + +#hide this tring to not confuse yetus +AUTHOR="uthor" +AUTHOR="@a${AUTHOR}" + +grep -r --include="*.java" "$AUTHOR" . +if grep -r --include="*.java" "$AUTHOR" .; then exit 0 else - exit -1 + exit 1 fi diff --git a/hadoop-ozone/dev-support/checks/build.sh b/hadoop-ozone/dev-support/checks/build.sh index 6a7811e..1197330 100755 --- a/hadoop-ozone/dev-support/checks/build.sh +++ b/hadoop-ozone/dev-support/checks/build.sh @@ -13,6 +13,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../../.." || exit 1 + export MAVEN_OPTS="-Xmx4096m" -mvn -am -pl :hadoop-ozone-dist -P hdds -Dmaven.javadoc.skip=true -DskipTests clean install +mvn -B -f pom.ozone.xml -Dmaven.javadoc.skip=true -DskipTests clean install exit $? diff --git a/hadoop-ozone/dev-support/checks/checkstyle.sh b/hadoop-ozone/dev-support/checks/checkstyle.sh index 0d80fbc..c4de528 100755 --- a/hadoop-ozone/dev-support/checks/checkstyle.sh +++ b/hadoop-ozone/dev-support/checks/checkstyle.sh @@ -13,11 +13,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -mvn -fn checkstyle:check -am -pl :hadoop-ozone-dist -Phdds +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../../.." || exit 1 + +mvn -B -fn checkstyle:check -f pom.ozone.xml + +#Print out the exact violations with parsing XML results with sed +find "." -name checkstyle-errors.xml -print0 | xargs -0 sed '$!N; /<file.*\n<\/file/d;P;D' | sed '/<\/.*/d;/<checkstyle.*/d;s/<error.*line="\([[:digit:]]*\)".*message="\([^"]\+\).*/ \1: \2/;s/<file name="\([^"]*\)".*/\1/;/<\?xml.*>/d' violations=$(grep -r error --include checkstyle-errors.xml .| wc -l) if [[ $violations -gt 0 ]]; then echo "There are $violations checkstyle violations" - exit -1 + exit 1 fi exit 0 diff --git a/hadoop-ozone/dev-support/checks/findbugs.sh b/hadoop-ozone/dev-support/checks/findbugs.sh index 1328492..545ad9f 100755 --- a/hadoop-ozone/dev-support/checks/findbugs.sh +++ b/hadoop-ozone/dev-support/checks/findbugs.sh @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../../.." || exit 1 FINDBUGS_ALL_FILE=./target/findbugs-all.txt @@ -20,15 +22,15 @@ mkdir -p ./target rm "$FINDBUGS_ALL_FILE" || true touch "$FINDBUGS_ALL_FILE" -mvn -fn findbugs:check -Dfindbugs.failOnError=false -am -pl :hadoop-ozone-dist -Phdds +mvn -B compile -fn findbugs:check -Dfindbugs.failOnError=false -f pom.ozone.xml -find hadoop-ozone -name findbugsXml.xml | xargs -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}" -find hadoop-hdds -name findbugsXml.xml | xargs -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}" +find hadoop-ozone -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}" +find hadoop-hdds -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}" -bugs=$(cat "$FINDBUGS_ALL_FILE" | wc -l) +bugs=$(wc -l < "$FINDBUGS_ALL_FILE") if [[ ${bugs} -gt 0 ]]; then - exit -1 + exit 1 else exit 0 fi diff --git a/hadoop-ozone/dev-support/checks/unit.sh b/hadoop-ozone/dev-support/checks/integration.sh similarity index 72% copy from hadoop-ozone/dev-support/checks/unit.sh copy to hadoop-ozone/dev-support/checks/integration.sh index d839f22..1aea3d1 100755 --- a/hadoop-ozone/dev-support/checks/unit.sh +++ b/hadoop-ozone/dev-support/checks/integration.sh @@ -13,12 +13,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../../.." || exit 1 + export MAVEN_OPTS="-Xmx4096m" -mvn -fn test -am -pl :hadoop-ozone-dist -P hdds -module_failed_tests=$(find "." -name 'TEST*.xml'\ - | xargs "grep" -l -E "<failure|<error"\ +mvn -B install -f pom.ozone.xml -DskipTests +mvn -B -fn test -f pom.ozone.xml -pl :hadoop-ozone-integration-test,:hadoop-ozone-filesystem +module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \ + | xargs -0 -n1 "grep" -l -E "<failure|<error"\ | awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml",""); print $NF}') if [[ -n "${module_failed_tests}" ]] ; then - exit -1 + exit 1 fi exit 0 diff --git a/hadoop-ozone/dev-support/checks/isolation.sh b/hadoop-ozone/dev-support/checks/isolation.sh index c8f2c62..1280235 100755 --- a/hadoop-ozone/dev-support/checks/isolation.sh +++ b/hadoop-ozone/dev-support/checks/isolation.sh @@ -13,12 +13,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -hadooplines=$(git diff --name-only HEAD~1..HEAD | grep -v hadoop-ozone | grep -v hadoop-hdds | wc -l ) +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../../.." || exit 1 + +hadooplines=$(git diff --name-only HEAD~1..HEAD | grep -v hadoop-ozone | grep -c -v hadoop-hdds ) if [ "$hadooplines" == "0" ]; then echo "Only ozone/hdds subprojects are changed" exit 0 else echo "Main hadoop projects are changed in an ozone patch." echo "Please do it in a HADOOP/HDFS patch and test it with hadoop precommit tests" - exit -1 + exit 1 fi diff --git a/hadoop-ozone/dev-support/checks/rat.sh b/hadoop-ozone/dev-support/checks/rat.sh index aadb251..68ca56e 100755 --- a/hadoop-ozone/dev-support/checks/rat.sh +++ b/hadoop-ozone/dev-support/checks/rat.sh @@ -13,12 +13,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../../.." || exit 1 mkdir -p target -rm target/rat-aggregated.txt -mvn -fn org.apache.rat:apache-rat-plugin:0.13:check -am -pl :hadoop-ozone-dist -Phdds -grep -r --include=rat.txt "!????" | tee ./target/rat-aggregated.txt +REPORT_FILE="$DIR/../../../target/rat-aggretaged.txt" +mkdir -p "$(dirname "$REPORT_FILE")" + +cd hadoop-hdds || exit 1 +mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check +cd ../hadoop-ozone || exit 1 +mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check + +cd "$DIR/../../.." || exit 1 +grep -r --include=rat.txt "!????" | tee "$REPORT_FILE" if [ "$(cat target/rat-aggregated.txt)" ]; then - exit -1 + exit 1 fi diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/shellcheck.sh similarity index 64% copy from hadoop-ozone/dev-support/checks/acceptance.sh copy to hadoop-ozone/dev-support/checks/shellcheck.sh index 8de920f..1284acd 100755 --- a/hadoop-ozone/dev-support/checks/acceptance.sh +++ b/hadoop-ozone/dev-support/checks/shellcheck.sh @@ -14,6 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -export HADOOP_VERSION=3 -"$DIR/../../../hadoop-ozone/dist/target/ozone-*-SNAPSHOT/compose/test-all.sh" -exit $? +cd "$DIR/../../.." || exit 1 + +OUTPUT_FILE="$DIR/../../../target/shell-problems.txt" +mkdir -p "$(dirname "$OUTPUT_FILE")" +echo "" > "$OUTPUT_FILE" +find "./hadoop-hdds" -type f -executable | grep -v target | grep -v node_modules | grep -v py | xargs -n1 shellcheck | tee "$OUTPUT_FILE" +find "./hadoop-ozone" -type f -executable | grep -v target | grep -v node_modules | grep -v py | xargs -n1 shellcheck | tee "$OUTPUT_FILE" + + +if [ "$(cat "$OUTPUT_FILE")" ]; then + exit 1 +fi diff --git a/hadoop-ozone/dev-support/checks/unit.sh b/hadoop-ozone/dev-support/checks/unit.sh index d839f22..1c6f2e9 100755 --- a/hadoop-ozone/dev-support/checks/unit.sh +++ b/hadoop-ozone/dev-support/checks/unit.sh @@ -14,11 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. export MAVEN_OPTS="-Xmx4096m" -mvn -fn test -am -pl :hadoop-ozone-dist -P hdds -module_failed_tests=$(find "." -name 'TEST*.xml'\ - | xargs "grep" -l -E "<failure|<error"\ +mvn -fn test -f pom.ozone.xml -pl \!:hadoop-ozone-integration-test,\!:hadoop-ozone-filesystem +module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \ + | xargs -n1 -0 "grep" -l -E "<failure|<error"\ | awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml",""); print $NF}') if [[ -n "${module_failed_tests}" ]] ; then - exit -1 + exit 1 fi exit 0 --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
