This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 22613e96 remove legacy comet-spark-shell (#1013)
22613e96 is described below
commit 22613e96890e70b9d5854dbb6b6fa9fb9d47ef94
Author: Andy Grove <[email protected]>
AuthorDate: Thu Oct 10 23:07:36 2024 -0600
remove legacy comet-spark-shell (#1013)
---
bin/comet-spark-shell | 86 ----------------------------
docs/source/contributor-guide/development.md | 2 -
2 files changed, 88 deletions(-)
diff --git a/bin/comet-spark-shell b/bin/comet-spark-shell
deleted file mode 100755
index 72100d2d..00000000
--- a/bin/comet-spark-shell
+++ /dev/null
@@ -1,86 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-set -e
-
-if [[ -z ${SPARK_HOME} ]]; then
- echo "ERROR: SPARK_HOME is not set: please set it to point to a Spark 3.4.0
distribution"
- exit 1
-fi
-
-POSITIONAL=()
-while [[ $# -gt 0 ]]; do
- key="$1"
-
- case $key in
- -r|--rebuild)
- REBUILD=YES
- shift
- ;;
- -o|--outdir)
- COMET_OUTDIR=$2
- shift
- shift
- ;;
- -d|--comet-dir)
- COMET_DIR=$2
- shift
- shift
- ;;
- *)
- POSITIONAL+=("$1")
- shift
- ;;
- esac
-done
-
-set -- "${POSITIONAL[@]}"
-
-COMET_DIR="${COMET_DIR:-$HOME/git/comet}"
-SPARK_VERSION_SHORT=3.4
-SCALA_BINARY_VERSION=2.12
-COMET_VERSION=0.1.0-SNAPSHOT
-COMET_SPARK_JAR=comet-spark-spark${SPARK_VERSION_SHORT}_${SCALA_BINARY_VERSION}-${COMET_VERSION}.jar
-COMET_OUTDIR="${COMET_OUTDIR:-/tmp}"
-
-if [[ ! -d $COMET_DIR ]]; then
- echo "Comet repo: $COMET_DIR, doesn't exist"
- exit 1
-fi
-
-if [[ ! -d $COMET_OUTDIR ]]; then
- echo "Output directory for Comet Spark library: $COMET_OUTDIR, doesn't exist"
- exit 1
-fi
-
-if [[ "X$REBUILD" == "XYES" ]]; then
- cd $COMET_DIR && make release
- cd $COMET_DIR/spark && cp target/${COMET_SPARK_JAR}
$COMET_OUTDIR/${COMET_SPARK_JAR}
-fi
-
-RUST_BACKTRACE=1 $SPARK_HOME/bin/spark-shell \
- --jars $COMET_OUTDIR/${COMET_SPARK_JAR} \
- --conf spark.sql.extensions=org.apache.comet.CometSparkSessionExtensions \
- --conf spark.comet.enabled=true \
- --conf spark.comet.exec.enabled=true \
- --conf spark.comet.exec.all.enabled=true \
- --conf spark.comet.exec.shuffle.enabled=true \
- --conf
spark.shuffle.manager=org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager
\
-$@
diff --git a/docs/source/contributor-guide/development.md
b/docs/source/contributor-guide/development.md
index 1d4b7f12..737c0aaf 100644
--- a/docs/source/contributor-guide/development.md
+++ b/docs/source/contributor-guide/development.md
@@ -47,8 +47,6 @@ A few common commands are specified in project's `Makefile`:
is useful when you want to test Comet local installation in another project
such as Spark.
- `make clean`: clean up the workspace
-- `bin/comet-spark-shell -d . -o spark/target/` run Comet spark shell for V1
datasources
-- `bin/comet-spark-shell -d . -o spark/target/ --conf
spark.sql.sources.useV1SourceList=""` run Comet spark shell for V2 datasources
## Development Environment
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
