This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch release-0.14.1-spark35-scala213 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 5676a8fb87a4337f9d8c5fb34a1e2946b83240ff Author: Lin Liu <[email protected]> AuthorDate: Tue Feb 13 15:20:37 2024 -0800 [MINOR] Disable Containers the Azure CI (#10662) This reverts commit 81cddbb77c2e555c044956d57e0b393f59c95ecc. --- .github/workflows/bot.yml | 6 +- Dockerfile | 31 ---- azure-pipelines-20230430.yml | 190 +++++++++++---------- .../hudi-metaserver/hudi-metaserver-server/pom.xml | 49 +++--- pom.xml | 1 - 5 files changed, 123 insertions(+), 154 deletions(-) diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index 6d9b7210f79..a82ba7d79fe 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -150,7 +150,7 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} FLINK_PROFILE: ${{ matrix.flinkProfile }} run: - ./mvnw clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -DskipTests=true -Phudi-platform-service -Pthrift-gen-source-with-script $MVN_ARGS -am -pl hudi-hadoop-mr,hudi-client/hudi-java-client + ./mvnw clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS -am -pl hudi-hadoop-mr,hudi-client/hudi-java-client - name: UT - hudi-hadoop-mr and hudi-client/hudi-java-client env: SCALA_PROFILE: ${{ matrix.scalaProfile }} @@ -335,10 +335,10 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} SCALA_PROFILE: 'scala-2.12' run: | - mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -Phudi-platform-service -Pthrift-gen-source-with-script + mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS # TODO remove the sudo below. It's a needed workaround as detailed in HUDI-5708. sudo chown -R "$USER:$(id -g -n)" hudi-platform-service/hudi-metaserver/target/generated-sources - mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version=1.10.0 -Phudi-platform-service -Pthrift-gen-source-with-script + mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version=1.10.0 - name: IT - Bundle Validation - OpenJDK 8 env: FLINK_PROFILE: ${{ matrix.flinkProfile }} diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index f8d03877143..00000000000 --- a/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Use a home made image as the base, which includes: -# utuntu:latest -# git -# thrift -# maven -# java8 -# Use an official Ubuntu base image -FROM apachehudi/hudi-ci-bundle-validation-base:azure_ci_test_base_new - -CMD ["java", "-version"] - -# Set the working directory to /app -WORKDIR /hudi - -# Copy git repo into the working directory -COPY . /hudi \ No newline at end of file diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml index c3e6b5c49ec..00bde22e9d9 100644 --- a/azure-pipelines-20230430.yml +++ b/azure-pipelines-20230430.yml @@ -98,7 +98,7 @@ parameters: variables: BUILD_PROFILES: '-Dscala-2.12 -Dspark3.2 -Dflink1.17' PLUGIN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true -ntp -B -V -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn' - MVN_OPTS_INSTALL: '-DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5' + MVN_OPTS_INSTALL: '-Phudi-platform-service -DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5' MVN_OPTS_TEST: '-fae -Pwarn-log $(BUILD_PROFILES) $(PLUGIN_OPTS)' JOB1_MODULES: ${{ join(',',parameters.job1Modules) }} JOB2_MODULES: ${{ join(',',parameters.job2Modules) }} @@ -108,120 +108,128 @@ variables: stages: - stage: test - variables: - - name: DOCKER_BUILDKIT - value: 1 jobs: - job: UT_FT_1 displayName: UT FT common & flink & UT client/spark-client timeoutInMinutes: '150' steps: - - task: Docker@2 - displayName: "login to docker" + - task: Maven@4 + displayName: maven install inputs: - command: "login" - containerRegistry: "apachehudi-docker-hub" - - task: Docker@2 - displayName: "load repo into image" + mavenPomFile: 'pom.xml' + goals: 'clean install' + options: $(MVN_OPTS_INSTALL) + publishJUnitResults: true + testResultsFiles: '**/surefire-reports/TEST-*.xml' + jdkVersionOption: '1.8' + - task: Maven@4 + displayName: UT common flink client/spark-client inputs: - containerRegistry: 'apachehudi-docker-hub' - repository: 'apachehudi/hudi-ci-bundle-validation-base' - command: 'build' - Dockerfile: '**/Dockerfile' - ImageName: $(Build.BuildId) - - task: Docker@2 - displayName: "UT FT common flink client/spark-client" + mavenPomFile: 'pom.xml' + goals: 'test' + options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB1_MODULES),hudi-client/hudi-spark-client + publishJUnitResults: true + testResultsFiles: '**/surefire-reports/TEST-*.xml' + jdkVersionOption: '1.8' + mavenOptions: '-Xmx4g' + - task: Maven@4 + displayName: FT common flink inputs: - containerRegistry: 'apachehudi-docker-hub' - repository: 'apachehudi/hudi-ci-bundle-validation-base' - command: 'run' - arguments: > - -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId) - /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL) - && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB1_MODULES),hudi-client/hudi-spark-client - && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB1_MODULES) - && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100" + mavenPomFile: 'pom.xml' + goals: 'test' + options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB1_MODULES) + publishJUnitResults: true + testResultsFiles: '**/surefire-reports/TEST-*.xml' + jdkVersionOption: '1.8' + mavenOptions: '-Xmx4g' + - script: | + grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 + displayName: Top 100 long-running testcases - job: UT_FT_2 displayName: FT client/spark-client & hudi-spark-datasource/hudi-spark timeoutInMinutes: '150' steps: - - task: Docker@2 - displayName: "login to docker" + - task: Maven@4 + displayName: maven install inputs: - command: "login" - containerRegistry: "apachehudi-docker-hub" - - task: Docker@2 - displayName: "load repo into image" + mavenPomFile: 'pom.xml' + goals: 'clean install' + options: $(MVN_OPTS_INSTALL) + publishJUnitResults: true + testResultsFiles: '**/surefire-reports/TEST-*.xml' + jdkVersionOption: '1.8' + - task: Maven@4 + displayName: FT client/spark-client & hudi-spark-datasource/hudi-spark inputs: - containerRegistry: 'apachehudi-docker-hub' - repository: 'apachehudi/hudi-ci-bundle-validation-base' - command: 'build' - Dockerfile: '**/Dockerfile' - ImageName: $(Build.BuildId) - - task: Docker@2 - displayName: "FT client/spark-client & hudi-spark-datasource/hudi-spark" - inputs: - containerRegistry: 'apachehudi-docker-hub' - repository: 'apachehudi/hudi-ci-bundle-validation-base' - command: 'run' - arguments: > - -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId) - /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL) - && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB2_MODULES) - && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100" + mavenPomFile: 'pom.xml' + goals: 'test' + options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB2_MODULES) + publishJUnitResults: true + testResultsFiles: '**/surefire-reports/TEST-*.xml' + jdkVersionOption: '1.8' + mavenOptions: '-Xmx4g' + - script: | + grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 + displayName: Top 100 long-running testcases - job: UT_FT_3 displayName: UT spark-datasource timeoutInMinutes: '240' steps: - - task: Docker@2 - displayName: "login to docker" - inputs: - command: "login" - containerRegistry: "apachehudi-docker-hub" - - task: Docker@2 - displayName: "load repo into image" + - task: Maven@4 + displayName: maven install inputs: - containerRegistry: 'apachehudi-docker-hub' - repository: 'apachehudi/hudi-ci-bundle-validation-base' - command: 'build' - Dockerfile: '**/Dockerfile' - ImageName: $(Build.BuildId) - - task: Docker@2 - displayName: "UT spark-datasource" + mavenPomFile: 'pom.xml' + goals: 'clean install' + options: $(MVN_OPTS_INSTALL) + publishJUnitResults: true + testResultsFiles: '**/surefire-reports/TEST-*.xml' + jdkVersionOption: '1.8' + - task: Maven@4 + displayName: UT spark-datasource inputs: - containerRegistry: 'apachehudi-docker-hub' - repository: 'apachehudi/hudi-ci-bundle-validation-base' - command: 'run' - arguments: > - -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId) - /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL) && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB3_MODULES) - && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100" + mavenPomFile: 'pom.xml' + goals: 'test' + options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB3_MODULES) + publishJUnitResults: true + testResultsFiles: '**/surefire-reports/TEST-*.xml' + jdkVersionOption: '1.8' + mavenOptions: '-Xmx4g' + - script: | + grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 + displayName: Top 100 long-running testcases - job: UT_FT_4 displayName: UT FT other modules timeoutInMinutes: '240' steps: - - task: Docker@2 - displayName: "login to docker hub" + - task: Maven@4 + displayName: maven install inputs: - command: "login" - containerRegistry: "apachehudi-docker-hub" - - task: Docker@2 - displayName: "load repo into image" + mavenPomFile: 'pom.xml' + goals: 'clean install' + options: $(MVN_OPTS_INSTALL) + publishJUnitResults: true + testResultsFiles: '**/surefire-reports/TEST-*.xml' + jdkVersionOption: '1.8' + - task: Maven@4 + displayName: UT other modules inputs: - containerRegistry: 'apachehudi-docker-hub' - repository: 'apachehudi/hudi-ci-bundle-validation-base' - command: 'build' - Dockerfile: '**/Dockerfile' - ImageName: $(Build.BuildId) - - task: Docker@2 - displayName: "UT FT other modules" + mavenPomFile: 'pom.xml' + goals: 'test' + options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB4_UT_MODULES) + publishJUnitResults: true + testResultsFiles: '**/surefire-reports/TEST-*.xml' + jdkVersionOption: '1.8' + mavenOptions: '-Xmx4g' + - task: Maven@4 + displayName: FT other modules inputs: - containerRegistry: 'apachehudi-docker-hub' - repository: 'apachehudi/hudi-ci-bundle-validation-base' - command: 'run' - arguments: > - -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId) - /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL) -Phudi-platform-service -Pthrift-gen-source - && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB4_UT_MODULES) - && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB4_UT_MODULES) - && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100" \ No newline at end of file + mavenPomFile: 'pom.xml' + goals: 'test' + options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB4_FT_MODULES) + publishJUnitResults: true + testResultsFiles: '**/surefire-reports/TEST-*.xml' + jdkVersionOption: '1.8' + mavenOptions: '-Xmx4g' + - script: | + grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 + displayName: Top 100 long-running testcases diff --git a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/pom.xml b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/pom.xml index 39bfc244176..d593eae75ea 100644 --- a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/pom.xml +++ b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/pom.xml @@ -92,34 +92,6 @@ </plugins> </build> </profile> - <profile> - <id>thrift-gen-source-with-script</id> - <build> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>exec-maven-plugin</artifactId> - <version>1.6.0</version> - <executions> - <execution> - <id>thrift-install-and-generate-source</id> - <phase>generate-sources</phase> - <goals> - <goal>exec</goal> - </goals> - </execution> - </executions> - <configuration> - <executable>${project.parent.basedir}/src/main/thrift/bin/thrift_binary.sh</executable> - <arguments> - <argument>${thrift.install.env}</argument> - </arguments> - <skip>false</skip> - </configuration> - </plugin> - </plugins> - </build> - </profile> <profile> <id>m1-mac</id> <properties> @@ -136,6 +108,27 @@ <build> <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.6.0</version> + <executions> + <execution> + <id>thrift-install-and-generate-source</id> + <phase>generate-sources</phase> + <goals> + <goal>exec</goal> + </goals> + </execution> + </executions> + <configuration> + <executable>${project.parent.basedir}/src/main/thrift/bin/thrift_binary.sh</executable> + <arguments> + <argument>${thrift.install.env}</argument> + </arguments> + <skip>false</skip> + </configuration> + </plugin> <plugin> <groupId>org.jacoco</groupId> <artifactId>jacoco-maven-plugin</artifactId> diff --git a/pom.xml b/pom.xml index 9dd577d8a64..f6c57bdba7c 100644 --- a/pom.xml +++ b/pom.xml @@ -227,7 +227,6 @@ <springboot.version>2.7.3</springboot.version> <spring.shell.version>2.1.1</spring.shell.version> <snappy.version>1.1.8.3</snappy.version> - <thrift.executable>/usr/local/bin/thrift</thrift.executable> </properties> <scm>
