This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new ce89d09 [SPARK-26343][K8S] Try to speed up running local k8s
integration tests
ce89d09 is described below
commit ce89d09bdf9e290359e369e85472786ac6c6a181
Author: Holden Karau <[email protected]>
AuthorDate: Thu Mar 14 19:39:43 2019 -0700
[SPARK-26343][K8S] Try to speed up running local k8s integration tests
Speed up running k8s integration tests locally by allowing folks to skip
the tgz dist build and extraction
Run tests locally without a distribution of Spark, just a local build
Closes #23380 from
holdenk/SPARK-26343-Speed-up-running-the-kubernetes-integration-tests-locally.
Authored-by: Holden Karau <[email protected]>
Signed-off-by: Marcelo Vanzin <[email protected]>
---
.../scripts/setup-integration-test-env.sh | 36 ++++++++++++++--------
.../k8s/integrationtest/KubernetesSuite.scala | 12 ++++++--
2 files changed, 33 insertions(+), 15 deletions(-)
diff --git
a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
index 36e30d7..84c42cb 100755
---
a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
+++
b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
@@ -16,6 +16,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+set -ex
TEST_ROOT_DIR=$(git rev-parse --show-toplevel)
UNPACKED_SPARK_TGZ="$TEST_ROOT_DIR/target/spark-dist-unpacked"
IMAGE_TAG_OUTPUT_FILE="$TEST_ROOT_DIR/target/image-tag.txt"
@@ -58,50 +59,59 @@ while (( "$#" )); do
shift
done
-if [[ $SPARK_TGZ == "N/A" ]];
+rm -rf "$UNPACKED_SPARK_TGZ"
+if [[ $SPARK_TGZ == "N/A" && $IMAGE_TAG == "N/A" ]];
then
- echo "Must specify a Spark tarball to build Docker images against with
--spark-tgz." && exit 1;
+ # If there is no spark image tag to test with and no src dir, build from
current
+ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd
)"
+ SPARK_INPUT_DIR="$(cd "$SCRIPT_DIR/"../../../../ >/dev/null 2>&1 && pwd )"
+
DOCKER_FILE_BASE_PATH="$SPARK_INPUT_DIR/resource-managers/kubernetes/docker/src/main/dockerfiles/spark"
+elif [[ $IMAGE_TAG == "N/A" ]];
+then
+ # If there is a test src tarball and no image tag we will want to build from
that
+ mkdir -p $UNPACKED_SPARK_TGZ
+ tar -xzvf $SPARK_TGZ --strip-components=1 -C $UNPACKED_SPARK_TGZ;
+ SPARK_INPUT_DIR="$UNPACKED_SPARK_TGZ"
+ DOCKER_FILE_BASE_PATH="$SPARK_INPUT_DIR/kubernetes/dockerfiles/spark"
fi
-rm -rf $UNPACKED_SPARK_TGZ
-mkdir -p $UNPACKED_SPARK_TGZ
-tar -xzvf $SPARK_TGZ --strip-components=1 -C $UNPACKED_SPARK_TGZ;
+# If there is a specific Spark image skip building and extraction/copy
if [[ $IMAGE_TAG == "N/A" ]];
then
IMAGE_TAG=$(uuidgen);
- cd $UNPACKED_SPARK_TGZ
+ cd $SPARK_INPUT_DIR
# Build PySpark image
- LANGUAGE_BINDING_BUILD_ARGS="-p
$UNPACKED_SPARK_TGZ/kubernetes/dockerfiles/spark/bindings/python/Dockerfile"
+ LANGUAGE_BINDING_BUILD_ARGS="-p
$DOCKER_FILE_BASE_PATH/bindings/python/Dockerfile"
- # Build SparkR image
- LANGUAGE_BINDING_BUILD_ARGS="$LANGUAGE_BINDING_BUILD_ARGS -R
$UNPACKED_SPARK_TGZ/kubernetes/dockerfiles/spark/bindings/R/Dockerfile"
+ # Build SparkR image -- disabled since this fails, re-enable as part of
SPARK-25152
+ # LANGUAGE_BINDING_BUILD_ARGS="$LANGUAGE_BINDING_BUILD_ARGS -R
$DOCKER_FILE_BASE_PATH/bindings/R/Dockerfile"
case $DEPLOY_MODE in
cloud)
# Build images
- $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t
$IMAGE_TAG $LANGUAGE_BINDING_BUILD_ARGS build
+ $SPARK_INPUT_DIR/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG
$LANGUAGE_BINDING_BUILD_ARGS build
# Push images appropriately
if [[ $IMAGE_REPO == gcr.io* ]] ;
then
gcloud docker -- push $IMAGE_REPO/spark:$IMAGE_TAG
else
- $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t
$IMAGE_TAG push
+ $SPARK_INPUT_DIR/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG
push
fi
;;
docker-for-desktop)
# Only need to build as this will place it in our local Docker repo
which is all
# we need for Docker for Desktop to work so no need to also push
- $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t
$IMAGE_TAG $LANGUAGE_BINDING_BUILD_ARGS build
+ $SPARK_INPUT_DIR/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG
$LANGUAGE_BINDING_BUILD_ARGS build
;;
minikube)
# Only need to build and if we do this with the -m option for minikube
we will
# build the images directly using the minikube Docker daemon so no need
to push
- $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -m -r $IMAGE_REPO -t
$IMAGE_TAG $LANGUAGE_BINDING_BUILD_ARGS build
+ $SPARK_INPUT_DIR/bin/docker-image-tool.sh -m -r $IMAGE_REPO -t
$IMAGE_TAG $LANGUAGE_BINDING_BUILD_ARGS build
;;
*)
echo "Unrecognized deploy mode $DEPLOY_MODE" && exit 1
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
index f8f4b41..16b8714 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
@@ -103,8 +103,16 @@ class KubernetesSuite extends SparkFunSuite
System.clearProperty(key)
}
- val sparkDirProp = System.getProperty(CONFIG_KEY_UNPACK_DIR)
- require(sparkDirProp != null, "Spark home directory must be provided in
system properties.")
+ val possible_spark_dirs = List(
+ // If someone specified the tgz for the tests look at the extraction dir
+ System.getProperty(CONFIG_KEY_UNPACK_DIR),
+ // Try the spark test home
+ sys.props("spark.test.home")
+ )
+ val sparkDirProp = possible_spark_dirs.filter(x =>
+ new File(Paths.get(x).toFile,
"bin/spark-submit").exists).headOption.getOrElse(null)
+ require(sparkDirProp != null,
+ s"Spark home directory must be provided in system properties tested
$possible_spark_dirs")
sparkHomeDir = Paths.get(sparkDirProp)
require(sparkHomeDir.toFile.isDirectory,
s"No directory found for spark home specified at $sparkHomeDir.")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
