This is an automated email from the ASF dual-hosted git repository.
elserj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ratis.git
The following commit(s) were added to refs/heads/master by this push:
new 2d487ae RATIS-672. Vagrant test's Maven 404's
2d487ae is described below
commit 2d487ae51552ac7417c59882c9ed4011db03249a
Author: Clay Baenziger <[email protected]>
AuthorDate: Tue Oct 1 20:43:15 2019 -0700
RATIS-672. Vagrant test's Maven 404's
Trying to get Clay's patch working -- call out to binary_locations.sh
before launching Yetus to build the docker image.
Re-application after revert. Needed to make sure binary_locations.sh
is invoked before building Docker so that PreCommit doesn't fail.
Signed-off-by: Josh Elser <[email protected]>
Co-authored-by: Josh Elser <[email protected]>
---
dev-support/binary_locations.sh | 32 ++++++++++++++++++++++++++++++++
dev-support/ci/common.sh | 2 ++
dev-support/docker/Dockerfile | 28 +++++++++++++++++++++-------
dev-support/vagrant/Vagrantfile | 31 +++++++++++++++++++++++++------
start-build-env.sh | 7 ++++++-
5 files changed, 86 insertions(+), 14 deletions(-)
diff --git a/dev-support/binary_locations.sh b/dev-support/binary_locations.sh
new file mode 100644
index 0000000..71b4afe
--- /dev/null
+++ b/dev-support/binary_locations.sh
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# List of binary artifacts used by the Docker and Vagrant test code
+# Provides a central point of URLs and checksums for all builds
+
+# Maven Location
+export ratis_maven_file="apache-maven-3.6.2-bin.tar.gz"
+export
ratis_maven_url="https://archive.apache.org/dist/maven/maven-3/3.6.2/binaries/${ratis_maven_file}";
+export
ratis_maven_sum="d941423d115cd021514bfd06c453658b1b3e39e6240969caf4315ab7119a77299713f14b620fb2571a264f8dff2473d8af3cb47b05acf0036fc2553199a5c1ee"
+
+# Go Location
+export ratis_go_file="go1.13.linux-amd64.tar.gz"
+export ratis_go_url="https://dl.google.com/go/${ratis_go_file}";
+export
ratis_go_sum="68a2297eb099d1a76097905a2ce334e3155004ec08cdea85f24527be3c48e856"
+
+# ProtoC Location
+export ratis_protoc_file="protoc-3.5.0-linux-x86_64.zip"
+export
ratis_protoc_url="https://github.com/google/protobuf/releases/download/v3.5.0/${ratis_protoc_file}";
+export
ratis_protoc_sum="49aa98db1877dcb69e89c7d217bb70cb1678d2266c3172f817348f2b5aab1d6a"
diff --git a/dev-support/ci/common.sh b/dev-support/ci/common.sh
index a786524..da6564b 100644
--- a/dev-support/ci/common.sh
+++ b/dev-support/ci/common.sh
@@ -32,6 +32,8 @@ mkdir -p "${ARTIFACTS}" "${YETUSDIR}" "${TOOLS}"
cp $BASEDIR/dev-support/yetus-personality.sh $WORKSPACE/
cp $BASEDIR/dev-support/docker/Dockerfile $WORKSPACE/
+source $BASEDIR/dev-support/binary_locations.sh
+
YETUS_VERSION=${YETUS_VERSION:-0.8.0}
echo "Downloading Yetus"
curl -L
https://archive.apache.org/dist/yetus/${YETUS_VERSION}/yetus-${YETUS_VERSION}-bin.tar.gz
-o yetus.tar.gz
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index 2cf298f..1f0ad8e 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -54,22 +54,36 @@ RUN apt-get -q install -y openjdk-8-jdk
######
# Install protobuf compiler
######
-
+ARG ratis_protoc_url
+ARG ratis_protoc_file
+ARG ratis_protoc_sum
RUN mkdir -p $HOME/protobuf && \
cd $HOME/protobuf && \
- wget
https://github.com/google/protobuf/releases/download/v3.5.0/protoc-3.5.0-linux-x86_64.zip
&& \
- unzip protoc-3.5.0-linux-x86_64.zip && \
+ wget --continue "${ratis_protoc_url:?Need \$ratis_protoc_url}" && \
+ if [ "$(sha256sum "${ratis_protoc_file:?Need \$ratis_protoc_file}")" != \
+ "${ratis_protoc_sum:?Need \$ratis_protoc_sum} $ratis_protoc_file" ];
then \
+ echo "Checksum failed for $ratis_protoc_file" > /dev/stderr && \
+ exit 1; \
+ fi && \
+ unzip "$ratis_protoc_file" && \
mv $HOME/protobuf/bin/protoc /usr/local/bin && \
chmod 755 /usr/local/bin/protoc
######
# Install Apache Maven
######
+ARG ratis_maven_url
+ARG ratis_maven_file
+ARG ratis_maven_sum
RUN mkdir -p /opt/maven && \
- curl -L -s -S \
-
http://www-us.apache.org/dist/maven/maven-3/3.6.2/binaries/apache-maven-3.6.2-bin.tar.gz
\
- -o /opt/maven.tar.gz && \
- tar xzf /opt/maven.tar.gz --strip-components 1 -C /opt/maven
+ cd $HOME && \
+ wget --continue "${ratis_maven_url:?Need \$ratis_maven_url}" && \
+ if [ "$(sha512sum "${ratis_maven_file:?Need \$ratis_maven_file}")" != \
+ "${ratis_maven_sum:?Need \$ratis_maven_sum} $ratis_maven_file" ]; then \
+ echo "Checksum failed for $ratis_maven_file" > /dev/stderr && \
+ exit 1; \
+ fi && \
+ tar xzf $ratis_maven_file --strip-components 1 -C /opt/maven
ENV MAVEN_HOME /opt/maven
ENV PATH "$PATH:/opt/maven/bin"
diff --git a/dev-support/vagrant/Vagrantfile b/dev-support/vagrant/Vagrantfile
index 78a00c8..bcf8476 100644
--- a/dev-support/vagrant/Vagrantfile
+++ b/dev-support/vagrant/Vagrantfile
@@ -78,6 +78,9 @@ Vagrant.configure('2') do |config|
# install packages
ratisbuild.vm.provision :shell, name: 'Install Packages', inline: <<-EOH
set -e
+ # load central file URLs and checksums
+ source #{RATIS_PATH}/dev-support/binary_locations.sh
+
# setup /usr/local/bin for non-packaged software
if [[ $(egrep -c 'PATH.*/usr/local/bin' /etc/environment) -eq 0 ]]; then
echo 'export PATH=${PATH}:/usr/local/bin' >> /etc/environment
@@ -89,14 +92,24 @@ Vagrant.configure('2') do |config|
# install Maven
mkdir -p /usr/local
- wget --continue
http://apache.mirrors.ionfish.org/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.tar.gz
- tar -xzf apache-maven-3.6.0-bin.tar.gz -C /usr/local
- [ -L /usr/local/bin/mvn ] || ln -s /usr/local/apache-maven-3.6.0/bin/mvn
/usr/local/bin/mvn
+ wget --continue "$ratis_maven_url"
+ if [[ "$(sha512sum "$ratis_maven_file")" != \
+ "$ratis_maven_sum $ratis_maven_file" ]]; then
+ echo "Checksum failed for $ratis_maven_file" > /dev/stderr
+ exit 1
+ fi
+ tar -xzf "$ratis_maven_file" -C /usr/local
+ [ -L /usr/local/bin/mvn ] || ln -s /usr/local/apache-maven-*/bin/mvn
/usr/local/bin/mvn
# Namazu dependencies
apt-get install -y git libnetfilter-queue-dev libzmq3-dev
- wget --continue https://dl.google.com/go/go1.11.2.linux-amd64.tar.gz
- tar -xzf go1.11.2.linux-amd64.tar.gz -C /usr/local
+ wget --continue "$ratis_go_url"
+ if [[ "$(sha256sum "$ratis_go_file")" != \
+ "$ratis_go_sum $ratis_go_file" ]]; then
+ echo "Checksum failed for $ratis_go_file" > /dev/stderr
+ exit 1
+ fi
+ tar -xzf "$ratis_go_file" -C /usr/local
[ -L /usr/local/bin/go ] || ln -s /usr/local/go/bin/go /usr/local/bin/go
EOH
@@ -109,6 +122,8 @@ Vagrant.configure('2') do |config|
export GOROOT=/usr/local/go
export GOPATH=`pwd`
# for some reason seelog fails to pull automatically
+ # Go should check its checksum database on go get per
+ # https://blog.golang.org/module-mirror-launch
go get -u github.com/cihub/seelog
./build
EOH
@@ -122,7 +137,7 @@ Vagrant.configure('2') do |config|
# or otherwise into /etc/environment
. /etc/environment
cd #{RATIS_PATH}
- mvn package -DskipTests
+ mvn clean install assembly:single -DskipTests=true
EOH
ratisbuild.vm.provider :virtualbox do |vb|
@@ -133,6 +148,10 @@ Vagrant.configure('2') do |config|
end
ratisbuild.vm.box = 'ubuntu/bionic64'
+ ratisbuild.vm.box_version = '20190918.0.0'
+# TODO: This seems to break vagrant destroy -f ratis-build
+# ratisbuild.vm.box_checksum_type = 'sha256'
+# ratisbuild.vm.box_checksum =
'ed0ac96a5ee41c8a2abff854c6ed1cefbc1cf60a53f1dfd283d66c27bf1c0d4a'
end
# Configure a generic VM with three Ratis servers
diff --git a/start-build-env.sh b/start-build-env.sh
old mode 100644
new mode 100755
index db182b3..5878007
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -19,7 +19,12 @@ set -e # exit on error
cd "$(dirname "$0")" # connect to root
-docker build -t ratis-build dev-support/docker
+# load URLs, checksums of dependencies
+source ./dev-support/binary_locations.sh
+# all dependency env. vars. start with ratis_ include them all via --build-arg
+build_args=$(env|awk 'BEGIN{FS="="};/^ratis_.*/{printf "--build-arg " $1 " "}')
+
+docker build $build_args -t ratis-build dev-support/docker
if [ "$(uname -s)" == "Linux" ]; then
USER_NAME=${SUDO_USER:=$USER}
