This is an automated email from the ASF dual-hosted git repository.
fgreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
The following commit(s) were added to refs/heads/master by this push:
new 4891600 SDAP-137 Webapp should have a mesos and a standalone image
(#51)
4891600 is described below
commit 4891600fd87d633a9c45d41fe600f7fc5cf07c9c
Author: fgreg <[email protected]>
AuthorDate: Wed Nov 7 08:45:04 2018 -0800
SDAP-137 Webapp should have a mesos and a standalone image (#51)
* Updating images.
* Updating images.
* ignore .nc files
* removed unused files.
* removed unused files.
* Updating images.
* ignore .nc files
* removed unused files.
* removed unused files.
* Updated bulid scripts and quickstart.
* workaround for basemap. change workdir of agent
* moved files to make more sense
* fixed error with conda home env var.
* don't remove spark package
* Need PROJ_LIB defined as agent env var.
---
analysis/.gitignore | 3 +-
docker/nexus-webapp/Readme.md | 41 ++++++
.../install_conda.sh} | 24 ++--
.../install_java.sh} | 24 +---
.../install_spark.sh} | 28 ++--
.../mesos/agent}/Dockerfile | 10 +-
.../mesos/agent}/docker-entrypoint.sh | 6 +-
docker/nexus-webapp/{ => mesos/base}/Dockerfile | 86 +++++++-----
.../mesos/base}/install_mesos.sh | 13 +-
.../mesos/master}/Dockerfile | 5 +-
.../mesos/webapp}/Dockerfile | 7 +-
.../{ => mesos/webapp}/docker-entrypoint.sh | 0
docker/nexus-webapp/{ => standalone}/Dockerfile | 42 +++---
.../{ => standalone}/docker-entrypoint.sh | 0
docker/spark-mesos-base/Dockerfile | 153 ---------------------
docs/quickstart.rst | 2 +-
16 files changed, 167 insertions(+), 277 deletions(-)
diff --git a/analysis/.gitignore b/analysis/.gitignore
index 2081316..559285c 100644
--- a/analysis/.gitignore
+++ b/analysis/.gitignore
@@ -1,3 +1,5 @@
+*.nc
+
# Created by .ignore support plugin (hsz.mobi)
### VirtualEnv template
# Virtualenv
@@ -194,4 +196,3 @@ com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
-
diff --git a/docker/nexus-webapp/Readme.md b/docker/nexus-webapp/Readme.md
new file mode 100644
index 0000000..7aeb962
--- /dev/null
+++ b/docker/nexus-webapp/Readme.md
@@ -0,0 +1,41 @@
+
+# How to Build
+
+All docker builds should happen from this directory. For copy/paste ability,
first export the environment variable `BUILD_VERSION` to the version number you
would like to tag images as.
+
+## spark-mesos-base
+
+ docker build -t sdap/spark-mesos-base:${BUILD_VERSION} -f
mesos/base/Dockerfile .
+
+## spark-mesos-master
+
+Builds from `spark-mesos-base` and supports `tag_version` build argument which
specifies the version of base to build from.
+
+ docker build -t sdap/spark-mesos-master:${BUILD_VERSION} -f
mesos/master/Dockerfile .
+
+## spark-mesos-agent
+
+Builds from `spark-mesos-base` and supports `tag_version` build argument which
specifies the version of base to build from.
+
+ docker build -t sdap/spark-mesos-agent:${BUILD_VERSION} -f
mesos/agent/Dockerfile .
+
+## nexus-webapp:mesos
+
+Builds from `spark-mesos-base` and supports `tag_version` build argument which
specifies the version of base to build from.
+
+ docker build -t sdap/nexus-webapp:mesos.${BUILD_VERSION} -f
mesos/webapp/Dockerfile .
+
+## nexus-webapp:standalone
+
+ docker build -t sdap/nexus-webapp:standalone.${BUILD_VERSION} -f
standalone/Dockerfile .
+
+# Push Images
+
+Push the images to the `sdap` organization on DockerHub
+
+ docker push sdap/spark-mesos-base:${BUILD_VERSION}
+ docker push sdap/spark-mesos-master:${BUILD_VERSION}
+ docker push sdap/spark-mesos-agent:${BUILD_VERSION}
+ docker push sdap/nexus-webapp:mesos.${BUILD_VERSION}
+ docker push sdap/nexus-webapp:standalone.${BUILD_VERSION}
+
diff --git a/docker/spark-mesos-base/install_nexusproto.sh
b/docker/nexus-webapp/install_conda.sh
similarity index 70%
copy from docker/spark-mesos-base/install_nexusproto.sh
copy to docker/nexus-webapp/install_conda.sh
index ce44c70..7d99ef2 100755
--- a/docker/spark-mesos-base/install_nexusproto.sh
+++ b/docker/nexus-webapp/install_conda.sh
@@ -13,23 +13,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-set -e
-APACHE_NEXUSPROTO="https://github.com/apache/incubator-sdap-nexusproto.git";
-MASTER="master"
+set -ebx
-GIT_REPO=${1:-$APACHE_NEXUSPROTO}
-GIT_BRANCH=${2:-$MASTER}
+URL=${1:-"https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh"}
+CONDA=${2:-"/usr/local/anaconda2"}
-mkdir nexusproto
-pushd nexusproto
-git init
-git pull ${GIT_REPO} ${GIT_BRANCH}
-
-./gradlew pythonInstall --info
-
-./gradlew install --info
-
-rm -rf /root/.gradle
+pushd /tmp
+wget -q ${URL} -O install_anaconda.sh
+/bin/bash install_anaconda.sh -b -p ${CONDA}
+rm install_anaconda.sh
+${CONDA}/bin/conda update -n base conda
popd
-rm -rf nexusproto
diff --git a/docker/spark-mesos-base/install_nexusproto.sh
b/docker/nexus-webapp/install_java.sh
similarity index 68%
copy from docker/spark-mesos-base/install_nexusproto.sh
copy to docker/nexus-webapp/install_java.sh
index ce44c70..b9ce92b 100755
--- a/docker/spark-mesos-base/install_nexusproto.sh
+++ b/docker/nexus-webapp/install_java.sh
@@ -13,23 +13,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-set -e
-APACHE_NEXUSPROTO="https://github.com/apache/incubator-sdap-nexusproto.git";
-MASTER="master"
+set -ebx
-GIT_REPO=${1:-$APACHE_NEXUSPROTO}
-GIT_BRANCH=${2:-$MASTER}
+URL=${1:-"http://download.oracle.com/otn-pub/java/jdk/8u191-b12/2787e4a523244c269598db4e85c51e0c/jdk-8u191-linux-x64.rpm"}
+RPM_PACKAGE=${URL##*/}
-mkdir nexusproto
-pushd nexusproto
-git init
-git pull ${GIT_REPO} ${GIT_BRANCH}
-
-./gradlew pythonInstall --info
-
-./gradlew install --info
-
-rm -rf /root/.gradle
-popd
-rm -rf nexusproto
+# Install Oracle JDK
+wget -q --no-cookies --no-check-certificate --header "Cookie:
gpw_e24=http%3A%2F%2Fwww.oracle.com%2F;
oraclelicense=accept-securebackup-cookie" ${URL}
+yum install -y ${RPM_PACKAGE}
+rm ${RPM_PACKAGE}
diff --git a/docker/spark-mesos-base/install_nexusproto.sh
b/docker/nexus-webapp/install_spark.sh
similarity index 64%
rename from docker/spark-mesos-base/install_nexusproto.sh
rename to docker/nexus-webapp/install_spark.sh
index ce44c70..1e668cc 100755
--- a/docker/spark-mesos-base/install_nexusproto.sh
+++ b/docker/nexus-webapp/install_spark.sh
@@ -13,23 +13,19 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-set -e
-APACHE_NEXUSPROTO="https://github.com/apache/incubator-sdap-nexusproto.git";
-MASTER="master"
+set -ebx
-GIT_REPO=${1:-$APACHE_NEXUSPROTO}
-GIT_BRANCH=${2:-$MASTER}
+DL_HOST=${1:-"http://d3kbcqa49mib13.cloudfront.net"}
+VERSION=${2:-"2.2.0"}
+DIR=${3:-"spark-${VERSION}"}
+INSTALL_DIR=${4:-"/usr/local"}
-mkdir nexusproto
-pushd nexusproto
-git init
-git pull ${GIT_REPO} ${GIT_BRANCH}
-
-./gradlew pythonInstall --info
-
-./gradlew install --info
-
-rm -rf /root/.gradle
+pushd ${INSTALL_DIR}
+wget --quiet ${DL_HOST}/spark-${VERSION}-bin-hadoop2.7.tgz
+tar -xzf spark-${VERSION}-bin-hadoop2.7.tgz
+chown -R root.root spark-${VERSION}-bin-hadoop2.7.tgz
+ln -s spark-${VERSION}-bin-hadoop2.7 ${DIR}
+# Do not remove the package, mesos requires it
+# rm spark-${VERSION}-bin-hadoop2.7.tgz
popd
-rm -rf nexusproto
diff --git a/docker/spark-mesos-agent/Dockerfile
b/docker/nexus-webapp/mesos/agent/Dockerfile
similarity index 64%
copy from docker/spark-mesos-agent/Dockerfile
copy to docker/nexus-webapp/mesos/agent/Dockerfile
index 471d63f..feca35f 100644
--- a/docker/spark-mesos-agent/Dockerfile
+++ b/docker/nexus-webapp/mesos/agent/Dockerfile
@@ -20,8 +20,12 @@ MAINTAINER Apache SDAP "[email protected]"
# Run a Mesos slave.
-WORKDIR ${MESOS_HOME}/build
+WORKDIR /tmp
-COPY docker-entrypoint.sh /tmp/docker-entrypoint.sh
+COPY mesos/agent/docker-entrypoint.sh /tmp/docker-entrypoint.sh
-ENTRYPOINT ["/tmp/docker-entrypoint.sh"]
+
+# This will run docker-entrypoint.sh with the value of CMD as default
arguments. However, if any arguments are supplied
+# to the docker run command when launching this image, the command line
arguments will override these CMD arguments
+ENTRYPOINT ["/bin/bash", "-c", "/tmp/docker-entrypoint.sh $(eval echo $@)",
"$@"]
+CMD ["--master=${MESOS_MASTER_NAME}:${MESOS_MASTER_PORT}",
"--port=${MESOS_AGENT_PORT}", "--work_dir=${MESOS_WORKDIR}"]
diff --git a/docker/spark-mesos-agent/docker-entrypoint.sh
b/docker/nexus-webapp/mesos/agent/docker-entrypoint.sh
similarity index 83%
rename from docker/spark-mesos-agent/docker-entrypoint.sh
rename to docker/nexus-webapp/mesos/agent/docker-entrypoint.sh
index 36d608b..2228b0d 100755
--- a/docker/spark-mesos-agent/docker-entrypoint.sh
+++ b/docker/nexus-webapp/mesos/agent/docker-entrypoint.sh
@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -e
+set -ebx
if [ -n "$TORNADO_HOST" ]; then
sed -i "s/server.socket_host = .*/server.socket_host = '${TORNADO_HOST}'/g"
${NEXUS_SRC}/analysis/webservice/config/web.ini
@@ -35,5 +35,5 @@ python setup.py install --force
cd ${NEXUS_SRC}/analysis
python setup.py install --force
-
-${MESOS_HOME}/build/bin/mesos-agent.sh
--master=${MESOS_MASTER_NAME}:${MESOS_MASTER_PORT} --port=${MESOS_AGENT_PORT}
--work_dir=${MESOS_WORKDIR} --no-systemd_enable_support --launcher=posix
--no-switch_user --executor_environment_variables='{ "PYTHON_EGG_CACHE": "/tmp"
}'
+# Set PROJ_LIB env var as workaround for missing environment variable for
basemap https://github.com/conda-forge/basemap-feedstock/issues/30
+${MESOS_HOME}/build/bin/mesos-agent.sh --no-systemd_enable_support
--launcher=posix --no-switch_user
--executor_environment_variables='{"PYTHON_EGG_CACHE": "/tmp",
"PROJ_LIB":"/usr/local/anaconda2/share/proj"}' "$@"
diff --git a/docker/nexus-webapp/Dockerfile
b/docker/nexus-webapp/mesos/base/Dockerfile
similarity index 51%
copy from docker/nexus-webapp/Dockerfile
copy to docker/nexus-webapp/mesos/base/Dockerfile
index 94ea293..2cce5d1 100644
--- a/docker/nexus-webapp/Dockerfile
+++ b/docker/nexus-webapp/mesos/base/Dockerfile
@@ -12,11 +12,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
FROM centos:7
MAINTAINER Apache SDAP "[email protected]"
+WORKDIR /tmp
+
RUN yum -y update && \
yum -y install \
bzip2 \
@@ -28,36 +29,58 @@ RUN yum -y update && \
which && \
yum clean all
-ARG SPARK_VERSION=2.2.0
-ENV SPARK_LOCAL_IP=127.0.0.1 \
- CASSANDRA_CONTACT_POINTS=127.0.0.1 \
- CASSANDRA_LOCAL_DATACENTER=datacenter1 \
- SOLR_URL_PORT=127.0.0.1:8983 \
+# Set environment variables. For Mesos, I used MESOS_VER because MESOS_VERSION
+# is expected to be a logical TRUE/FALSE flag that tells Mesos whether or not
+# to simply print the version number and exit.
+
+ENV INSTALL_LOC=/usr/local \
+ HADOOP_VERSION=2.7.3 \
+ SPARK_VERSION=2.2.0 \
+ MESOS_VER=1.5.1 \
+ MESOS_MASTER_PORT=5050 \
+ MESOS_AGENT_PORT=5051 \
+ MESOS_WORKDIR=/var/lib/mesos \
+ MESOS_IP=0.0.0.0 \
+ MESOS_MASTER_NAME=mesos-master \
+ PYTHON_EGG_CACHE=/tmp \
+ JAVA_HOME=/usr/java/default \
+ NEXUS_SRC=/tmp/incubator-sdap-nexus
+
+ENV CONDA_HOME=${INSTALL_LOC}/anaconda2 \
+ MESOS_HOME=${INSTALL_LOC}/mesos-${MESOS_VER} \
SPARK_DIR=spark-${SPARK_VERSION} \
SPARK_PACKAGE=spark-${SPARK_VERSION}-bin-hadoop2.7 \
- SPARK_HOME=/usr/local/spark-${SPARK_VERSION} \
- PYSPARK_DRIVER_PYTHON=/usr/local/anaconda2/bin/python \
- PYSPARK_PYTHON=/usr/local/anaconda2/bin/python \
+ MESOS_MASTER=mesos://${MESOS_IP}:${MESOS_PORT} \
+ MESOS_PACKAGE=mesos-${MESOS_VER}.tar.gz
+
+ENV SPARK_HOME=${INSTALL_LOC}/${SPARK_DIR} \
+ PYSPARK_DRIVER_PYTHON=${CONDA_HOME}/bin/python \
+ PYSPARK_PYTHON=${CONDA_HOME}/bin/python \
PYSPARK_SUBMIT_ARGS="--driver-memory=4g pyspark-shell" \
-
PYTHONPATH=${PYTHONPATH}:/usr/local/spark-${SPARK_VERSION}/python:/usr/local/spark-${SPARK_VERSION}/python/lib/py4j-0.10.4-src.zip:/usr/local/spark-${SPARK_VERSION}/python/lib/pyspark.zip
\
- SPARK_EXECUTOR_URI=/usr/local/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz \
- NEXUS_SRC=/tmp/incubator-sdap-nexus
+ # Workaround for missing environment variable for basemap
https://github.com/conda-forge/basemap-feedstock/issues/30
+ PROJ_LIB=${CONDA_HOME}/share/proj
+
+ENV
PYTHONPATH=${PYTHONPATH}:${SPARK_HOME}/python:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip:${SPARK_HOME}/python/lib/pyspark.zip
\
+ MESOS_NATIVE_JAVA_LIBRARY=${INSTALL_LOC}/lib/libmesos.so \
+ SPARK_EXECUTOR_URI=${INSTALL_LOC}/${SPARK_PACKAGE}.tgz \
+ PATH=${CONDA_HOME}/bin:${PATH}
+
+# Install Oracle JDK
+COPY install_java.sh ./install_java.sh
+RUN ./install_java.sh
"http://download.oracle.com/otn-pub/java/jdk/8u191-b12/2787e4a523244c269598db4e85c51e0c/jdk-8u191-linux-x64.rpm";
+
+# Set up Mesos
+COPY mesos/base/install_mesos.sh ./install_mesos.sh
+RUN source ./install_mesos.sh
# Install Spark
-RUN cd /usr/local && \
- wget --quiet
http://d3kbcqa49mib13.cloudfront.net/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz
&& \
- tar -xzf spark-${SPARK_VERSION}-bin-hadoop2.7.tgz && \
- chown -R root.root spark-${SPARK_VERSION}-bin-hadoop2.7.tgz && \
- ln -s spark-${SPARK_VERSION}-bin-hadoop2.7 ${SPARK_DIR} && \
- rm spark-${SPARK_VERSION}-bin-hadoop2.7.tgz && \
- cd /
+COPY install_spark.sh ./install_spark.sh
+RUN ./install_spark.sh "http://d3kbcqa49mib13.cloudfront.net"; ${SPARK_VERSION}
${SPARK_DIR} ${INSTALL_LOC}
+
+# Install Anaconda
+COPY install_conda.sh ./install_conda.sh
+RUN ./install_conda.sh
"https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh";
${CONDA_HOME}
-# Install Miniconda
-RUN wget -q
https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O
install_anaconda.sh && \
- /bin/bash install_anaconda.sh -b -p /usr/local/anaconda2 && \
- rm install_anaconda.sh && \
- /usr/local/anaconda2/bin/conda update -n base conda
-ENV PATH /usr/local/anaconda2/bin:$PATH
# Conda dependencies for nexus
RUN conda install -c conda-forge -y netCDF4 && \
conda install -y numpy cython mpld3 scipy basemap gdal matplotlib && \
@@ -70,22 +93,15 @@ RUN conda install -c conda-forge -y netCDF4 && \
ln -s libhdf5_cpp.so.12 libhdf5_cpp.so.10 && \
ln -s libjpeg.so.9 libjpeg.so.8
-# Install Oracle JDK 1.8u182-b13
-RUN wget -q --no-cookies --no-check-certificate --header "Cookie:
gpw_e24=http%3A%2F%2Fwww.oracle.com%2F;
oraclelicense=accept-securebackup-cookie"
"http://download.oracle.com/otn-pub/java/jdk/8u181-b13/96a7b8442fe848ef90c96a2fad6ed6d1/jdk-8u181-linux-x64.rpm";
&& \
- yum install -y jdk-8u181-linux-x64.rpm && \
- rm jdk-8u181-linux-x64.rpm
-
-COPY *.sh /tmp/
-
# Install nexusproto and nexus
ARG APACHE_NEXUSPROTO=https://github.com/apache/incubator-sdap-nexusproto.git
ARG APACHE_NEXUSPROTO_BRANCH=master
ARG APACHE_NEXUS=https://github.com/apache/incubator-sdap-nexus.git
ARG APACHE_NEXUS_BRANCH=master
ARG REBUILD_CODE=1
+COPY install_nexusproto.sh ./install_nexusproto.sh
+COPY install_nexus.sh ./install_nexus.sh
RUN /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH &&
\
/tmp/install_nexus.sh $APACHE_NEXUS $APACHE_NEXUS_BRANCH $NEXUS_SRC
-EXPOSE 8083
-
-ENTRYPOINT ["/tmp/docker-entrypoint.sh"]
+CMD ["/bin/bash"]
diff --git a/docker/spark-mesos-base/install_mesos.sh
b/docker/nexus-webapp/mesos/base/install_mesos.sh
old mode 100644
new mode 100755
similarity index 92%
rename from docker/spark-mesos-base/install_mesos.sh
rename to docker/nexus-webapp/mesos/base/install_mesos.sh
index 655d737..fc0c475
--- a/docker/spark-mesos-base/install_mesos.sh
+++ b/docker/nexus-webapp/mesos/base/install_mesos.sh
@@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+set -ebx
+
# Install a few utility tools
yum install -y tar wget git
@@ -56,10 +58,15 @@ tar -zxf ${MESOS_PACKAGE} -C ${INSTALL_LOC}
rm -f ${MESOS_PACKAGE}
# Configure and build.
-cd ${MESOS_HOME}
+mkdir -p ${MESOS_WORKDIR}
+mkdir -p ${MESOS_HOME}
+pushd ${MESOS_HOME}
mkdir build
-cd build
+pushd build
../configure
make
-make check
+# Can't run make check until this is fixed:
https://issues.apache.org/jira/browse/MESOS-8608
+# make check
make install
+popd
+popd
diff --git a/docker/spark-mesos-master/Dockerfile
b/docker/nexus-webapp/mesos/master/Dockerfile
similarity index 69%
rename from docker/spark-mesos-master/Dockerfile
rename to docker/nexus-webapp/mesos/master/Dockerfile
index c1d7d39..a6f7f91 100644
--- a/docker/spark-mesos-master/Dockerfile
+++ b/docker/nexus-webapp/mesos/master/Dockerfile
@@ -24,4 +24,7 @@ EXPOSE ${MESOS_MASTER_PORT}
WORKDIR ${MESOS_HOME}/build
-CMD ["/bin/bash", "-c", "./bin/mesos-master.sh --ip=${MESOS_IP}
--hostname=${MESOS_MASTER_NAME} --port=${MESOS_MASTER_PORT}
--work_dir=${MESOS_WORKDIR}"]
+# This will run mesos-master.sh with the value of CMD as default arguments.
However, if any arguments are supplied
+# to the docker run command when launching this image, the command line
arguments will override these CMD arguments
+ENTRYPOINT ["/bin/bash", "-c", "./bin/mesos-master.sh $(eval echo $@)", "$@"]
+CMD ["--ip=${MESOS_IP}", "--hostname=${MESOS_MASTER_NAME}",
"--port=${MESOS_MASTER_PORT}", "--work_dir=${MESOS_WORKDIR}"]
diff --git a/docker/spark-mesos-agent/Dockerfile
b/docker/nexus-webapp/mesos/webapp/Dockerfile
similarity index 88%
rename from docker/spark-mesos-agent/Dockerfile
rename to docker/nexus-webapp/mesos/webapp/Dockerfile
index 471d63f..aaa64b2 100644
--- a/docker/spark-mesos-agent/Dockerfile
+++ b/docker/nexus-webapp/mesos/webapp/Dockerfile
@@ -18,10 +18,7 @@ FROM sdap/spark-mesos-base:${tag_version}
MAINTAINER Apache SDAP "[email protected]"
-# Run a Mesos slave.
-
-WORKDIR ${MESOS_HOME}/build
-
-COPY docker-entrypoint.sh /tmp/docker-entrypoint.sh
+COPY /mesos/webapp/docker-entrypoint.sh /tmp/docker-entrypoint.sh
+ENV MASTER=mesos://${MESOS_MASTER_NAME}:${MESOS_MASTER_PORT}
ENTRYPOINT ["/tmp/docker-entrypoint.sh"]
diff --git a/docker/nexus-webapp/docker-entrypoint.sh
b/docker/nexus-webapp/mesos/webapp/docker-entrypoint.sh
similarity index 100%
copy from docker/nexus-webapp/docker-entrypoint.sh
copy to docker/nexus-webapp/mesos/webapp/docker-entrypoint.sh
diff --git a/docker/nexus-webapp/Dockerfile
b/docker/nexus-webapp/standalone/Dockerfile
similarity index 71%
rename from docker/nexus-webapp/Dockerfile
rename to docker/nexus-webapp/standalone/Dockerfile
index 94ea293..06073fc 100644
--- a/docker/nexus-webapp/Dockerfile
+++ b/docker/nexus-webapp/standalone/Dockerfile
@@ -41,23 +41,25 @@ ENV SPARK_LOCAL_IP=127.0.0.1 \
PYSPARK_SUBMIT_ARGS="--driver-memory=4g pyspark-shell" \
PYTHONPATH=${PYTHONPATH}:/usr/local/spark-${SPARK_VERSION}/python:/usr/local/spark-${SPARK_VERSION}/python/lib/py4j-0.10.4-src.zip:/usr/local/spark-${SPARK_VERSION}/python/lib/pyspark.zip
\
SPARK_EXECUTOR_URI=/usr/local/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz \
- NEXUS_SRC=/tmp/incubator-sdap-nexus
+ NEXUS_SRC=/tmp/incubator-sdap-nexus \
+ PATH=/usr/local/anaconda2/bin:$PATH \
+ # Workaround for missing environment variable for basemap
https://github.com/conda-forge/basemap-feedstock/issues/30
+ PROJ_LIB=/usr/local/anaconda2/share/proj
+
+WORKDIR /tmp
+
+# Install Oracle JDK
+COPY install_java.sh ./install_java.sh
+RUN ./install_java.sh
"http://download.oracle.com/otn-pub/java/jdk/8u191-b12/2787e4a523244c269598db4e85c51e0c/jdk-8u191-linux-x64.rpm";
# Install Spark
-RUN cd /usr/local && \
- wget --quiet
http://d3kbcqa49mib13.cloudfront.net/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz
&& \
- tar -xzf spark-${SPARK_VERSION}-bin-hadoop2.7.tgz && \
- chown -R root.root spark-${SPARK_VERSION}-bin-hadoop2.7.tgz && \
- ln -s spark-${SPARK_VERSION}-bin-hadoop2.7 ${SPARK_DIR} && \
- rm spark-${SPARK_VERSION}-bin-hadoop2.7.tgz && \
- cd /
+COPY install_spark.sh ./install_spark.sh
+RUN ./install_spark.sh "http://d3kbcqa49mib13.cloudfront.net"; ${SPARK_VERSION}
${SPARK_DIR} ${INSTALL_LOC}
+
+# Install Anaconda
+COPY install_conda.sh ./install_conda.sh
+RUN ./install_conda.sh
"https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh";
/usr/local/anaconda2
-# Install Miniconda
-RUN wget -q
https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O
install_anaconda.sh && \
- /bin/bash install_anaconda.sh -b -p /usr/local/anaconda2 && \
- rm install_anaconda.sh && \
- /usr/local/anaconda2/bin/conda update -n base conda
-ENV PATH /usr/local/anaconda2/bin:$PATH
# Conda dependencies for nexus
RUN conda install -c conda-forge -y netCDF4 && \
conda install -y numpy cython mpld3 scipy basemap gdal matplotlib && \
@@ -70,22 +72,16 @@ RUN conda install -c conda-forge -y netCDF4 && \
ln -s libhdf5_cpp.so.12 libhdf5_cpp.so.10 && \
ln -s libjpeg.so.9 libjpeg.so.8
-# Install Oracle JDK 1.8u182-b13
-RUN wget -q --no-cookies --no-check-certificate --header "Cookie:
gpw_e24=http%3A%2F%2Fwww.oracle.com%2F;
oraclelicense=accept-securebackup-cookie"
"http://download.oracle.com/otn-pub/java/jdk/8u181-b13/96a7b8442fe848ef90c96a2fad6ed6d1/jdk-8u181-linux-x64.rpm";
&& \
- yum install -y jdk-8u181-linux-x64.rpm && \
- rm jdk-8u181-linux-x64.rpm
-
-COPY *.sh /tmp/
-
# Install nexusproto and nexus
ARG APACHE_NEXUSPROTO=https://github.com/apache/incubator-sdap-nexusproto.git
ARG APACHE_NEXUSPROTO_BRANCH=master
ARG APACHE_NEXUS=https://github.com/apache/incubator-sdap-nexus.git
ARG APACHE_NEXUS_BRANCH=master
ARG REBUILD_CODE=1
+COPY install_nexusproto.sh ./install_nexusproto.sh
+COPY install_nexus.sh ./install_nexus.sh
RUN /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH &&
\
/tmp/install_nexus.sh $APACHE_NEXUS $APACHE_NEXUS_BRANCH $NEXUS_SRC
-EXPOSE 8083
-
+COPY standalone/docker-entrypoint.sh ./docker-entrypoint.sh
ENTRYPOINT ["/tmp/docker-entrypoint.sh"]
diff --git a/docker/nexus-webapp/docker-entrypoint.sh
b/docker/nexus-webapp/standalone/docker-entrypoint.sh
similarity index 100%
rename from docker/nexus-webapp/docker-entrypoint.sh
rename to docker/nexus-webapp/standalone/docker-entrypoint.sh
diff --git a/docker/spark-mesos-base/Dockerfile
b/docker/spark-mesos-base/Dockerfile
deleted file mode 100644
index 4c27bd6..0000000
--- a/docker/spark-mesos-base/Dockerfile
+++ /dev/null
@@ -1,153 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-FROM centos:7
-
-MAINTAINER Apache SDAP "[email protected]"
-
-WORKDIR /tmp
-
-RUN yum -y update && \
- yum -y install wget \
- git \
- which \
- bzip2 \
- gcc \
- python-devel
-
-# Install Oracle JDK 1.8u121-b13
-RUN wget -q --no-cookies --no-check-certificate --header "Cookie:
gpw_e24=http%3A%2F%2Fwww.oracle.com%2F;
oraclelicense=accept-securebackup-cookie"
"http://download.oracle.com/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441/jdk-8u121-linux-x64.rpm";
&& \
- yum -y install jdk-8u121-linux-x64.rpm && \
- rm jdk-8u121-linux-x64.rpm
-ENV JAVA_HOME /usr/java/default
-
-# ########################
-# # Apache Maven #
-# ########################
-ENV M2_HOME /usr/local/apache-maven
-ENV M2 $M2_HOME/bin
-ENV PATH $PATH:$M2
-
-RUN mkdir $M2_HOME && \
- wget -q
http://mirror.stjschools.org/public/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
&& \
- tar -xvzf apache-maven-3.3.9-bin.tar.gz -C $M2_HOME --strip-components=1
&& \
- rm -f apache-maven-3.3.9-bin.tar.gz
-
-# ########################
-# # Anaconda #
-# ########################
-RUN wget -q https://repo.continuum.io/archive/Anaconda2-4.3.0-Linux-x86_64.sh
-O install_anaconda.sh && \
- /bin/bash install_anaconda.sh -b -p /usr/local/anaconda2 && \
- rm install_anaconda.sh
-ENV PATH $PATH:/usr/local/anaconda2/bin
-
-# Set environment variables. For Mesos, I used MESOS_VER because MESOS_VERSION
-# is expected to be a logical TRUE/FALSE flag that tells Mesos whether or not
-# to simply print the version number and exit.
-
-ENV INSTALL_LOC=/usr/local \
- HADOOP_VERSION=2.7.3 \
- SPARK_VERSION=2.1.0 \
- MESOS_VER=1.5.0 \
- MESOS_MASTER_PORT=5050 \
- MESOS_AGENT_PORT=5051 \
- MESOS_WORKDIR=/var/lib/mesos \
- MESOS_IP=0.0.0.0 \
- MESOS_MASTER_NAME=mesos-master \
- PYTHON_EGG_CACHE=/tmp
-
-ENV CONDA_HOME=${INSTALL_LOC}/anaconda2 \
- MESOS_HOME=${INSTALL_LOC}/mesos-${MESOS_VER} \
- SPARK_DIR=spark-${SPARK_VERSION} \
- SPARK_PACKAGE=spark-${SPARK_VERSION}-bin-hadoop2.7 \
- MESOS_MASTER=mesos://${MESOS_IP}:${MESOS_PORT} \
- MESOS_PACKAGE=mesos-${MESOS_VER}.tar.gz
-
-ENV SPARK_HOME=${INSTALL_LOC}/${SPARK_DIR} \
- PYSPARK_DRIVER_PYTHON=${CONDA_HOME}/bin/python \
- PYSPARK_PYTHON=${CONDA_HOME}/bin/python \
- PYSPARK_SUBMIT_ARGS="--driver-memory=4g pyspark-shell"
-
-ENV
PYTHONPATH=${PYTHONPATH}:${SPARK_HOME}/python:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip:${SPARK_HOME}/python/lib/pyspark.zip
\
- MESOS_NATIVE_JAVA_LIBRARY=${INSTALL_LOC}/lib/libmesos.so \
- SPARK_EXECUTOR_URI=${INSTALL_LOC}/${SPARK_PACKAGE}.tgz
-
-WORKDIR ${INSTALL_LOC}
-
-# Set up Spark
-
-RUN wget --quiet http://d3kbcqa49mib13.cloudfront.net/${SPARK_PACKAGE}.tgz && \
- tar -xzf ${SPARK_PACKAGE}.tgz && \
- chown -R root.root ${SPARK_PACKAGE} && \
- ln -s ${SPARK_PACKAGE} ${SPARK_DIR}
-
-# Set up Mesos
-
-COPY install_mesos.sh .
-
-RUN source ./install_mesos.sh && \
- mkdir ${MESOS_WORKDIR}
-
-# Set up Anaconda environment
-
-ENV PATH=${CONDA_HOME}/bin:${PATH}:${HADOOP_HOME}/bin:${SPARK_HOME}/bin
-
-RUN conda install -c conda-forge -y netCDF4 && \
- conda install -y numpy cython mpld3 scipy basemap gdal matplotlib && \
- pip install shapely cassandra-driver==3.5.0 && \
- conda install -c conda-forge backports.functools_lru_cache=1.3
-
-# Workaround missing libcom_err.so (needed for gdal)
-
-RUN cd /usr/lib64 && ln -s libcom_err.so.2 libcom_err.so.3
-
-# Workaround missing conda libs needed for gdal
-
-RUN cd ${CONDA_HOME}/lib && \
- ln -s libnetcdf.so.11 libnetcdf.so.7 && \
- ln -s libkea.so.1.4.6 libkea.so.1.4.5 && \
- ln -s libhdf5_cpp.so.12 libhdf5_cpp.so.10 && \
- ln -s libjpeg.so.9 libjpeg.so.8
-
-RUN yum install -y mesa-libGL.x86_64
-
-# Install nexusproto
-ARG APACHE_NEXUSPROTO=https://github.com/apache/incubator-sdap-nexusproto.git
-ARG APACHE_NEXUSPROTO_BRANCH=master
-ARG REBUILD_CODE=1
-COPY install_nexusproto.sh ./install_nexusproto.sh
-RUN ./install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH
-
-# Retrieve NEXUS code and build it.
-
-WORKDIR /
-
-RUN git clone https://github.com/apache/incubator-sdap-nexus.git
-
-RUN sed -i 's/,webservice.algorithms.doms//g'
/incubator-sdap-nexus/analysis/webservice/config/web.ini && \
- sed -i 's/127.0.0.1/nexus-webapp/g'
/incubator-sdap-nexus/analysis/webservice/config/web.ini && \
- sed -i
's/127.0.0.1/cassandra1,cassandra2,cassandra3,cassandra4,cassandra5,cassandra6/g'
/incubator-sdap-nexus/data-access/nexustiles/config/datastores.ini && \
- sed -i 's/localhost:8983/solr1:8983/g'
/incubator-sdap-nexus/data-access/nexustiles/config/datastores.ini
-
-WORKDIR /incubator-sdap-nexus/data-access
-
-RUN python setup.py install
-
-WORKDIR /incubator-sdap-nexus/analysis
-
-RUN python setup.py install
-
-WORKDIR /tmp
-
-CMD ["/bin/bash"]
diff --git a/docs/quickstart.rst b/docs/quickstart.rst
index 42a403d..c03dec1 100644
--- a/docs/quickstart.rst
+++ b/docs/quickstart.rst
@@ -45,7 +45,7 @@ Pull the necessary Docker images from the `SDAP repository
<https://hub.docker.c
docker pull sdap/ningester:${VERSION}
docker pull sdap/solr-singlenode:${VERSION}
docker pull sdap/cassandra:${VERSION}
- docker pull sdap/nexus-webapp:${VERSION}
+ docker pull sdap/nexus-webapp:standalone.${VERSION}
.. _quickstart-step2:
