This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new f0b6245 [SPARK-27158][BUILD] dev/mima and dev/scalastyle support
dynamic profiles
f0b6245 is described below
commit f0b6245ea4cef0dc61d4277f1d6874ccf315e47a
Author: Yuming Wang <[email protected]>
AuthorDate: Fri Mar 15 08:20:42 2019 +0900
[SPARK-27158][BUILD] dev/mima and dev/scalastyle support dynamic profiles
## What changes were proposed in this pull request?
`dev/mima` and `dev/scalastyle` support dynamic reading profiles from
`modules.py`.
## How was this patch tested?
manual tests
Closes #24089 from wangyum/SPARK-27158.
Authored-by: Yuming Wang <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
dev/lint-scala | 2 +-
dev/mima | 2 +-
dev/run-tests.py | 14 ++++++++++----
dev/sbt-checkstyle | 4 ++--
dev/scalastyle | 12 ++++--------
5 files changed, 18 insertions(+), 16 deletions(-)
diff --git a/dev/lint-scala b/dev/lint-scala
index c676dfd..9c701ab 100755
--- a/dev/lint-scala
+++ b/dev/lint-scala
@@ -20,4 +20,4 @@
SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
SPARK_ROOT_DIR="$(dirname $SCRIPT_DIR)"
-"$SCRIPT_DIR/scalastyle"
+"$SCRIPT_DIR/scalastyle" "$1"
diff --git a/dev/mima b/dev/mima
index dc7b085..f324c5c 100755
--- a/dev/mima
+++ b/dev/mima
@@ -24,7 +24,7 @@ set -e
FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
cd "$FWDIR"
-SPARK_PROFILES="-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl
-Phive-thriftserver -Phive"
+SPARK_PROFILES=${1:-"-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl
-Pkinesis-asl -Phive-thriftserver -Phive"}
TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export
tools/fullClasspath" | tail -n1)"
OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES
"export oldDeps/fullClasspath" | tail -n1)"
diff --git a/dev/run-tests.py b/dev/run-tests.py
index aa106af..535c877 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -175,9 +175,11 @@ def run_apache_rat_checks():
run_cmd([os.path.join(SPARK_HOME, "dev", "check-license")])
-def run_scala_style_checks():
+def run_scala_style_checks(build_profiles):
set_title_and_block("Running Scala style checks", "BLOCK_SCALA_STYLE")
- run_cmd([os.path.join(SPARK_HOME, "dev", "lint-scala")])
+ profiles = " ".join(build_profiles)
+ print("[info] Checking Scala style using SBT with these profiles: ",
profiles)
+ run_cmd([os.path.join(SPARK_HOME, "dev", "lint-scala"), profiles])
def run_java_style_checks(build_profiles):
@@ -359,7 +361,10 @@ def build_apache_spark(build_tool, hadoop_version):
def detect_binary_inop_with_mima(hadoop_version):
build_profiles = get_hadoop_profiles(hadoop_version) +
modules.root.build_profile_flags
set_title_and_block("Detecting binary incompatibilities with MiMa",
"BLOCK_MIMA")
- run_cmd([os.path.join(SPARK_HOME, "dev", "mima")] + build_profiles)
+ profiles = " ".join(build_profiles)
+ print("[info] Detecting binary incompatibilities with MiMa using SBT with
these profiles: ",
+ profiles)
+ run_cmd([os.path.join(SPARK_HOME, "dev", "mima"), profiles])
def run_scala_tests_maven(test_profiles):
@@ -582,7 +587,8 @@ def main():
if not changed_files or any(f.endswith(".scala")
or f.endswith("scalastyle-config.xml")
for f in changed_files):
- run_scala_style_checks()
+ build_profiles = get_hadoop_profiles(hadoop_version) +
modules.root.build_profile_flags
+ run_scala_style_checks(build_profiles)
should_run_java_style_checks = False
if not changed_files or any(f.endswith(".java")
or f.endswith("checkstyle.xml")
diff --git a/dev/sbt-checkstyle b/dev/sbt-checkstyle
index 1ecad59..415155f 100755
--- a/dev/sbt-checkstyle
+++ b/dev/sbt-checkstyle
@@ -17,12 +17,12 @@
# limitations under the License.
#
-profiles=${1:-"-Pkinesis-asl -Pmesos -Pkubernetes -Pyarn -Phive
-Phive-thriftserver"}
+SPARK_PROFILES=${1:-"-Pkinesis-asl -Pmesos -Pkubernetes -Pyarn -Phive
-Phive-thriftserver"}
# NOTE: echo "q" is needed because SBT prompts the user for input on
encountering a build file
# with failure (either resolution or compilation); the "q" makes SBT quit.
ERRORS=$(echo -e "q\n" \
- | build/sbt ${profiles} checkstyle test:checkstyle \
+ | build/sbt ${SPARK_PROFILES} checkstyle test:checkstyle \
| awk '{if($1~/error/)print}' \
)
diff --git a/dev/scalastyle b/dev/scalastyle
index d5cebb3..212ef90 100755
--- a/dev/scalastyle
+++ b/dev/scalastyle
@@ -17,19 +17,15 @@
# limitations under the License.
#
+SPARK_PROFILES=${1:-"-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl
-Pkinesis-asl -Phive-thriftserver -Phive"}
+
# NOTE: echo "q" is needed because SBT prompts the user for input on
encountering a build file
# with failure (either resolution or compilation); the "q" makes SBT quit.
ERRORS=$(echo -e "q\n" \
| build/sbt \
- -Pkinesis-asl \
- -Pmesos \
- -Pkubernetes \
- -Pyarn \
- -Phive \
- -Phive-thriftserver \
- -Pspark-ganglia-lgpl \
+ ${SPARK_PROFILES} \
-Pdocker-integration-tests \
- -Pkubernetes-integration-tests \
+ -Pkubernetes-integration-tests \
scalastyle test:scalastyle \
| awk '{if($1~/error/)print}' \
)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
