This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a7fe726 HDDS-4000. Split acceptance tests to reduce CI feedback time
(#1236)
a7fe726 is described below
commit a7fe7260f5a72809fe5387725c92942c454ce899
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Mon Jul 27 13:46:22 2020 +0200
HDDS-4000. Split acceptance tests to reduce CI feedback time (#1236)
---
.github/workflows/post-commit.yml | 17 ++++---
hadoop-ozone/dev-support/checks/acceptance.sh | 2 +
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 +
.../dist/src/main/compose/ozonesecure/test.sh | 2 +
hadoop-ozone/dist/src/main/compose/test-all.sh | 31 +++++++++---
pom.xml | 55 +---------------------
6 files changed, 42 insertions(+), 67 deletions(-)
diff --git a/.github/workflows/post-commit.yml
b/.github/workflows/post-commit.yml
index 9588423..e472aea 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -123,6 +123,13 @@ jobs:
acceptance:
name: acceptance
runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ suite:
+ - secure
+ - unsecure
+ - misc
+ fail-fast: false
steps:
- uses: actions/cache@v2
with:
@@ -154,12 +161,13 @@ jobs:
cd /mnt/ozone && hadoop-ozone/dev-support/checks/acceptance.sh
env:
KEEP_IMAGE: false
+ OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }}
OZONE_WITH_COVERAGE: true
OZONE_VOLUME_OWNER: 1000
- uses: actions/upload-artifact@master
if: always()
with:
- name: acceptance
+ name: acceptance-${{ matrix.suite }}
path: /mnt/ozone/target/acceptance
continue-on-error: true
- run: |
@@ -170,16 +178,11 @@ jobs:
integration:
name: integration
runs-on: ubuntu-18.04
- needs:
- - build
strategy:
matrix:
profile:
- client
- - filesystem
- - filesystem-contract
- - freon
- - hdds-om
+ - filesystem-hdds
- ozone
fail-fast: false
steps:
diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh
b/hadoop-ozone/dev-support/checks/acceptance.sh
index d95c034..99d8d52 100755
--- a/hadoop-ozone/dev-support/checks/acceptance.sh
+++ b/hadoop-ozone/dev-support/checks/acceptance.sh
@@ -28,6 +28,8 @@ fi
mkdir -p "$REPORT_DIR"
+export OZONE_ACCEPTANCE_SUITE
+
cd "$DIST_DIR/compose" || exit 1
./test-all.sh
RES=$?
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
index c40339e..b5b778f 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
@@ -15,6 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+#suite:unsecure
+
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
export COMPOSE_DIR
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
index 84de2a9..076b83a 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test.sh
@@ -15,6 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+#suite:secure
+
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
export COMPOSE_DIR
diff --git a/hadoop-ozone/dist/src/main/compose/test-all.sh
b/hadoop-ozone/dist/src/main/compose/test-all.sh
index e7f6f71..da3b80e 100755
--- a/hadoop-ozone/dist/src/main/compose/test-all.sh
+++ b/hadoop-ozone/dist/src/main/compose/test-all.sh
@@ -31,21 +31,40 @@ if [ "$OZONE_WITH_COVERAGE" ]; then
export
HADOOP_OPTS="-javaagent:share/coverage/jacoco-agent.jar=output=tcpclient,address=$DOCKER_BRIDGE_IP,includes=org.apache.hadoop.ozone.*:org.apache.hadoop.hdds.*:org.apache.hadoop.fs.ozone.*"
fi
+if [[ -n "${OZONE_ACCEPTANCE_SUITE}" ]]; then
+ tests=$(find "$SCRIPT_DIR" -name test.sh | xargs grep -l
"^#suite:${OZONE_ACCEPTANCE_SUITE}$" | sort)
+
+ # 'misc' is default suite, add untagged tests, too
+ if [[ "misc" == "${OZONE_ACCEPTANCE_SUITE}" ]]; then
+ untagged="$(find "$SCRIPT_DIR" -name test.sh | xargs grep -L "^#suite:")"
+ if [[ -n "${untagged}" ]]; then
+ tests=$(echo ${tests} ${untagged} | xargs -n1 | sort)
+ fi
+ fi
+
+ if [[ -z "${tests}" ]]; then
+ echo "No tests found for suite ${OZONE_ACCEPTANCE_SUITE}"
+ exit 1
+ fi
+else
+ tests=$(find "$SCRIPT_DIR" -name test.sh | grep "${OZONE_TEST_SELECTOR:-""}"
| sort)
+fi
+
RESULT=0
-IFS=$'\n'
# shellcheck disable=SC2044
-for test in $(find "$SCRIPT_DIR" -name test.sh | grep
"${OZONE_TEST_SELECTOR:-""}" |sort); do
- echo "Executing test in $(dirname "$test")"
+for t in ${tests}; do
+ d="$(dirname "${t}")"
+ echo "Executing test in ${d}"
#required to read the .env file from the right location
- cd "$(dirname "$test")" || continue
+ cd "${d}" || continue
./test.sh
ret=$?
if [[ $ret -ne 0 ]]; then
RESULT=1
- echo "ERROR: Test execution of $(dirname "$test") is FAILED!!!!"
+ echo "ERROR: Test execution of ${d} is FAILED!!!!"
fi
- RESULT_DIR="$(dirname "$test")/result"
+ RESULT_DIR="${d}/result"
cp "$RESULT_DIR"/robot-*.xml "$RESULT_DIR"/docker-*.log "$RESULT_DIR"/*.out*
"$ALL_RESULT_DIR"/
done
diff --git a/pom.xml b/pom.xml
index aa62f5d..969e5c6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2147,22 +2147,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
</build>
</profile>
<profile>
- <id>freon</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <includes>
- <include>org.apache.hadoop.ozone.freon.**</include>
- </includes>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- <profile>
<id>client</id>
<build>
<plugins>
@@ -2179,23 +2163,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
</build>
</profile>
<profile>
- <id>filesystem-contract</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <includes>
- <include>org.apache.hadoop.fs.ozone.contract.**</include>
- </includes>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- <profile>
- <id>filesystem</id>
+ <id>filesystem-hdds</id>
<build>
<plugins>
<plugin>
@@ -2204,26 +2172,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
<configuration>
<includes>
<include>org.apache.hadoop.fs.ozone.**</include>
- </includes>
- <excludes>
- <exclude>org.apache.hadoop.fs.ozone.contract.**</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- <profile>
- <id>hdds-om</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <includes>
<include>org.apache.hadoop.hdds.**</include>
- <include>org.apache.hadoop.ozone.om.**</include>
</includes>
</configuration>
</plugin>
@@ -2243,8 +2192,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
</includes>
<excludes>
<exclude>org.apache.hadoop.ozone.client.**</exclude>
- <exclude>org.apache.hadoop.ozone.freon.**</exclude>
- <exclude>org.apache.hadoop.ozone.om.**</exclude>
</excludes>
</configuration>
</plugin>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
