This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new e861d01 bring back dev scripts for ballista (#648)
e861d01 is described below
commit e861d017ba1b9ee17c1f45d390fc9926e5af7c37
Author: Jiayu Liu <[email protected]>
AuthorDate: Thu Jul 1 04:39:32 2021 +0800
bring back dev scripts for ballista (#648)
---
dev/build-ballista-docker-arm64.sh | 34 ++++++++++++
dev/build-ballista-docker.sh | 24 +++++++++
dev/build-set-env.sh | 20 +++++++
dev/build-ui.sh | 23 ++++++++
dev/release/rat_exclude_files.txt | 108 +++++++++++++++++++++++++++++++++++++
dev/update_arrow_deps.py | 83 ++++++++++++++++++++++++++++
6 files changed, 292 insertions(+)
diff --git a/dev/build-ballista-docker-arm64.sh
b/dev/build-ballista-docker-arm64.sh
new file mode 100755
index 0000000..5d95177
--- /dev/null
+++ b/dev/build-ballista-docker-arm64.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if [ -z "${DOCKER_REPO}" ]; then
+ echo "DOCKER_REPO env var must be set"
+ exit -1
+fi
+cargo install cross
+cross build --release --target aarch64-unknown-linux-gnu
+rm -rf temp-ballista-docker
+mkdir temp-ballista-docker
+cp target/aarch64-unknown-linux-gnu/release/ballista-executor
temp-ballista-docker
+cp target/aarch64-unknown-linux-gnu/release/ballista-scheduler
temp-ballista-docker
+cp target/aarch64-unknown-linux-gnu/release/tpch temp-ballista-docker
+mkdir temp-ballista-docker/queries/
+cp benchmarks/queries/*.sql temp-ballista-docker/queries/
+docker buildx build --push -t $DOCKER_REPO/ballista-arm64
--platform=linux/arm64 -f dev/docker/ballista-arm64.Dockerfile
temp-ballista-docker
+rm -rf temp-ballista-docker
\ No newline at end of file
diff --git a/dev/build-ballista-docker.sh b/dev/build-ballista-docker.sh
new file mode 100755
index 0000000..bc028da
--- /dev/null
+++ b/dev/build-ballista-docker.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+. ./dev/build-set-env.sh
+docker build -t ballista-base:$BALLISTA_VERSION -f
dev/docker/ballista-base.dockerfile .
+docker build -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile .
diff --git a/dev/build-set-env.sh b/dev/build-set-env.sh
new file mode 100755
index 0000000..3eb29e7
--- /dev/null
+++ b/dev/build-set-env.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+export BALLISTA_VERSION=$(awk -F'[ ="]+' '$1 == "version" { print $2 }'
ballista/rust/core/Cargo.toml)
diff --git a/dev/build-ui.sh b/dev/build-ui.sh
new file mode 100755
index 0000000..0708397
--- /dev/null
+++ b/dev/build-ui.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+. ./dev/build-set-env.sh
+docker build -t ballista-scheduler-ui:$BALLISTA_VERSION -f
dev/docker/ballista-scheduler-ui.dockerfile ballista/ui/scheduler
diff --git a/dev/release/rat_exclude_files.txt
b/dev/release/rat_exclude_files.txt
new file mode 100644
index 0000000..96beccd
--- /dev/null
+++ b/dev/release/rat_exclude_files.txt
@@ -0,0 +1,108 @@
+*.npmrc
+*.gitignore
+*.dockerignore
+.gitmodules
+*_generated.h
+*_generated.js
+*_generated.ts
+*.csv
+*.json
+*.snap
+.github/ISSUE_TEMPLATE/*.md
+.github/pull_request_template.md
+ci/etc/rprofile
+ci/etc/*.patch
+ci/vcpkg/*.patch
+CHANGELOG.md
+dev/requirements*.txt
+dev/archery/MANIFEST.in
+dev/archery/requirements*.txt
+dev/archery/archery/tests/fixtures/*
+dev/archery/archery/crossbow/tests/fixtures/*
+dev/release/rat_exclude_files.txt
+dev/tasks/homebrew-formulae/apache-arrow.rb
+dev/tasks/linux-packages/apache-arrow-apt-source/debian/apache-arrow-apt-source.install
+dev/tasks/linux-packages/apache-arrow-apt-source/debian/compat
+dev/tasks/linux-packages/apache-arrow-apt-source/debian/control
+dev/tasks/linux-packages/apache-arrow-apt-source/debian/rules
+dev/tasks/linux-packages/apache-arrow-apt-source/debian/source/format
+dev/tasks/linux-packages/apache-arrow/debian/compat
+dev/tasks/linux-packages/apache-arrow/debian/control.in
+dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-1.0.install
+dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-cuda-1.0.install
+dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-dataset-1.0.install
+dev/tasks/linux-packages/apache-arrow/debian/gir1.2-gandiva-1.0.install
+dev/tasks/linux-packages/apache-arrow/debian/gir1.2-parquet-1.0.install
+dev/tasks/linux-packages/apache-arrow/debian/gir1.2-plasma-1.0.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-dev.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.doc-base
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.links
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib400.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-dev.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib-dev.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib400.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda400.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-dev.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-dev.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.doc-base
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.links
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib400.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset400.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-dev.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight400.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight400.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow-python400.install
+dev/tasks/linux-packages/apache-arrow/debian/libarrow400.install
+dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install
+dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-dev.install
+dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.doc-base
+dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.install
+dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.links
+dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib400.install
+dev/tasks/linux-packages/apache-arrow/debian/libgandiva400.install
+dev/tasks/linux-packages/apache-arrow/debian/libparquet-dev.install
+dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-dev.install
+dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.doc-base
+dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.install
+dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.links
+dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib400.install
+dev/tasks/linux-packages/apache-arrow/debian/libparquet400.install
+dev/tasks/linux-packages/apache-arrow/debian/libplasma-dev.install
+dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-dev.install
+dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.doc-base
+dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.install
+dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.links
+dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib400.install
+dev/tasks/linux-packages/apache-arrow/debian/libplasma400.install
+dev/tasks/linux-packages/apache-arrow/debian/patches/series
+dev/tasks/linux-packages/apache-arrow/debian/plasma-store-server.install
+dev/tasks/linux-packages/apache-arrow/debian/rules
+dev/tasks/linux-packages/apache-arrow/debian/source/format
+dev/tasks/linux-packages/apache-arrow/debian/watch
+dev/tasks/requirements*.txt
+dev/tasks/conda-recipes/*
+pax_global_header
+MANIFEST.in
+__init__.pxd
+__init__.py
+requirements.txt
+*.html
+*.sgml
+*.css
+*.png
+*.ico
+*.svg
+*.devhelp2
+*.scss
+.gitattributes
+rust-toolchain
+benchmarks/queries/q*.sql
+ballista/rust/scheduler/testdata/*
+ballista/ui/scheduler/yarn.lock
+python/rust-toolchain
+python/requirements*.txt
diff --git a/dev/update_arrow_deps.py b/dev/update_arrow_deps.py
new file mode 100755
index 0000000..44bdf42
--- /dev/null
+++ b/dev/update_arrow_deps.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Script that updates the arrow dependencies in datafusion and ballista, locall
+#
+# installation:
+# pip install tomlkit requests
+#
+# usage:
+# python update_arrow_deps.py
+
+from pathlib import Path
+
+# use tomlkit as it preserves comments and other formatting
+import tomlkit
+import requests
+
+
+# find latest arrow-rs sha
+def get_arrow_sha():
+ url = 'https://api.github.com/repos/apache/arrow-rs/branches/master'
+ response = requests.get(url)
+ return response.json()['commit']['sha']
+
+
+# Update all entries that look like
+# {
+# 'git': 'https://github.com/apache/arrow-rs',
+# 'rev': 'c3fe3bab9905739fdda75301dab07a18c91731bd'
+# }
+# to point at a new SHA
+def update_dependencies(dependencies, new_sha):
+ if dependencies is None:
+ return
+ for dep_name in dependencies:
+ dep = dependencies[dep_name]
+ if hasattr(dep, 'get'):
+ if dep.get('git') == 'https://github.com/apache/arrow-rs':
+ dep['rev'] = new_sha
+
+
+def update_cargo_toml(cargo_toml, new_sha):
+ print('updating {}'.format(cargo_toml.absolute()))
+ with open(cargo_toml) as f:
+ data = f.read()
+
+ doc = tomlkit.parse(data)
+
+ update_dependencies(doc.get('dependencies'), new_sha)
+ update_dependencies(doc.get('dev-dependencies'), new_sha)
+
+ with open(cargo_toml, 'w') as f:
+ f.write(tomlkit.dumps(doc))
+
+
+# Begin main script
+
+repo_root = Path(__file__).parent.parent.absolute()
+
+
+new_sha = get_arrow_sha()
+
+print('Updating files in {} to use sha {}'.format(repo_root, new_sha))
+
+
+for cargo_toml in repo_root.rglob('Cargo.toml'):
+ update_cargo_toml(cargo_toml, new_sha)
