This is an automated email from the ASF dual-hosted git repository. srowen pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new f399d0d [SPARK-37437][BUILD] Remove unused hive profile and related CI test f399d0d is described below commit f399d0d748dc72740ba0b3f662d658546215c71d Author: Angerszhuuuu <angers....@gmail.com> AuthorDate: Sat Nov 27 08:50:46 2021 -0600 [SPARK-37437][BUILD] Remove unused hive profile and related CI test ### What changes were proposed in this pull request? Since we only support hive-2.3, we should remove the unused profile and related GA test. `-Phive-2.3` still works after removing the profile ### Why are the changes needed? Remove unused profile ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Not need Closes #34679 from AngersZhuuuu/SPARK-37437. Authored-by: Angerszhuuuu <angers....@gmail.com> Signed-off-by: Sean Owen <sro...@gmail.com> --- dev/create-release/release-build.sh | 2 +- dev/run-tests-jenkins.py | 3 --- dev/run-tests.py | 23 +---------------------- dev/test-dependencies.sh | 8 +++----- pom.xml | 5 ----- 5 files changed, 5 insertions(+), 36 deletions(-) diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh index 96b8d4e..44baedd 100755 --- a/dev/create-release/release-build.sh +++ b/dev/create-release/release-build.sh @@ -192,7 +192,7 @@ SCALA_2_12_PROFILES="-Pscala-2.12" HIVE_PROFILES="-Phive -Phive-thriftserver" # Profiles for publishing snapshots and release to Maven Central # We use Apache Hive 2.3 for publishing -PUBLISH_PROFILES="$BASE_PROFILES $HIVE_PROFILES -Phive-2.3 -Pspark-ganglia-lgpl -Pkinesis-asl -Phadoop-cloud" +PUBLISH_PROFILES="$BASE_PROFILES $HIVE_PROFILES -Pspark-ganglia-lgpl -Pkinesis-asl -Phadoop-cloud" # Profiles for building binary releases BASE_RELEASE_PROFILES="$BASE_PROFILES -Psparkr" diff --git a/dev/run-tests-jenkins.py b/dev/run-tests-jenkins.py index f24e702..67d0972 100755 --- a/dev/run-tests-jenkins.py +++ b/dev/run-tests-jenkins.py @@ -174,9 +174,6 @@ def main(): os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop2.7" if "test-hadoop3.2" in ghprb_pull_title: os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop3.2" - # Switch the Hive profile based on the PR title: - if "test-hive2.3" in ghprb_pull_title: - os.environ["AMPLAB_JENKINS_BUILD_HIVE_PROFILE"] = "hive2.3" # Switch the Scala profile based on the PR title: if "test-scala2.13" in ghprb_pull_title: os.environ["AMPLAB_JENKINS_BUILD_SCALA_PROFILE"] = "scala2.13" diff --git a/dev/run-tests.py b/dev/run-tests.py index 55c65ed..25df8f6 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -345,24 +345,6 @@ def get_hadoop_profiles(hadoop_version): sys.exit(int(os.environ.get("CURRENT_BLOCK", 255))) -def get_hive_profiles(hive_version): - """ - For the given Hive version tag, return a list of Maven/SBT profile flags for - building and testing against that Hive version. - """ - - sbt_maven_hive_profiles = { - "hive2.3": ["-Phive-2.3"], - } - - if hive_version in sbt_maven_hive_profiles: - return sbt_maven_hive_profiles[hive_version] - else: - print("[error] Could not find", hive_version, "in the list. Valid options", - " are", sbt_maven_hive_profiles.keys()) - sys.exit(int(os.environ.get("CURRENT_BLOCK", 255))) - - def build_spark_maven(extra_profiles): # Enable all of the profiles for the build: build_profiles = extra_profiles + modules.root.build_profile_flags @@ -616,7 +598,6 @@ def main(): build_tool = os.environ.get("AMPLAB_JENKINS_BUILD_TOOL", "sbt") scala_version = os.environ.get("AMPLAB_JENKINS_BUILD_SCALA_PROFILE") hadoop_version = os.environ.get("AMPLAB_JENKINS_BUILD_PROFILE", "hadoop3.2") - hive_version = os.environ.get("AMPLAB_JENKINS_BUILD_HIVE_PROFILE", "hive2.3") test_env = "amplab_jenkins" # add path for Python3 in Jenkins if we're calling from a Jenkins machine # TODO(sknapp): after all builds are ported to the ubuntu workers, change this to be: @@ -627,14 +608,12 @@ def main(): build_tool = "sbt" scala_version = os.environ.get("SCALA_PROFILE") hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop3.2") - hive_version = os.environ.get("HIVE_PROFILE", "hive2.3") if "GITHUB_ACTIONS" in os.environ: test_env = "github_actions" else: test_env = "local" - extra_profiles = get_hadoop_profiles(hadoop_version) + get_hive_profiles(hive_version) + \ - get_scala_profiles(scala_version) + extra_profiles = get_hadoop_profiles(hadoop_version) + get_scala_profiles(scala_version) print("[info] Using build tool", build_tool, "with profiles", *(extra_profiles + ["under environment", test_env])) diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh index 156a0d3..e23a0b6 100755 --- a/dev/test-dependencies.sh +++ b/dev/test-dependencies.sh @@ -86,20 +86,18 @@ $MVN -q versions:set -DnewVersion=$TEMP_VERSION -DgenerateBackupPoms=false > /de for HADOOP_HIVE_PROFILE in "${HADOOP_HIVE_PROFILES[@]}"; do if [[ $HADOOP_HIVE_PROFILE == **hadoop-3.2-hive-2.3** ]]; then HADOOP_PROFILE=hadoop-3.2 - HIVE_PROFILE=hive-2.3 else HADOOP_PROFILE=hadoop-2.7 - HIVE_PROFILE=hive-2.3 fi echo "Performing Maven install for $HADOOP_HIVE_PROFILE" - $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE jar:jar jar:test-jar install:install clean -q + $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE jar:jar jar:test-jar install:install clean -q echo "Performing Maven validate for $HADOOP_HIVE_PROFILE" - $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE validate -q + $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE validate -q echo "Generating dependency manifest for $HADOOP_HIVE_PROFILE" mkdir -p dev/pr-deps - $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE dependency:build-classpath -pl assembly -am \ + $MVN $HADOOP_MODULE_PROFILES -P$HADOOP_PROFILE dependency:build-classpath -pl assembly -am \ | grep "Dependencies classpath:" -A 1 \ | tail -n 1 | tr ":" "\n" | awk -F '/' '{ # For each dependency classpath, we fetch the last three parts split by "/": artifact id, version, and jar name. diff --git a/pom.xml b/pom.xml index f8cc8bb..87e3489 100644 --- a/pom.xml +++ b/pom.xml @@ -3354,11 +3354,6 @@ </profile> <profile> - <id>hive-2.3</id> - <!-- Default hive profile. Uses global properties. --> - </profile> - - <profile> <id>yarn</id> <modules> <module>resource-managers/yarn</module> --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org