This is an automated email from the ASF dual-hosted git repository. fgreg pushed a commit to branch SDAP-207 in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit 5a453de0e6480112e33527e2cc66c6a260d41696 Author: Frank Greguska <[email protected]> AuthorDate: Wed Aug 21 18:55:29 2019 -0700 SDAP-207 fixed gdal issues --- docker/nexus-webapp/install_conda.sh | 15 ++++++++++++--- .../nexus-webapp/mesos/agent/docker-entrypoint.sh | 4 +++- docker/nexus-webapp/mesos/base/Dockerfile | 18 +++++++++--------- .../nexus-webapp/mesos/webapp/docker-entrypoint.sh | 2 ++ docker/nexus-webapp/standalone/Dockerfile | 21 +++++++++++---------- docker/nexus-webapp/standalone/docker-entrypoint.sh | 2 ++ 6 files changed, 39 insertions(+), 23 deletions(-) diff --git a/docker/nexus-webapp/install_conda.sh b/docker/nexus-webapp/install_conda.sh index 7d99ef2..b5d0a45 100755 --- a/docker/nexus-webapp/install_conda.sh +++ b/docker/nexus-webapp/install_conda.sh @@ -16,12 +16,21 @@ set -ebx -URL=${1:-"https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh"} -CONDA=${2:-"/usr/local/anaconda2"} +URL=${1:-"https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh"} +CONDA=${2:-"/usr/local/anaconda3"} + +CONDA_ENV_NAME=${CONDA_ENV_NAME:-"nexus-webapp"} pushd /tmp wget -q ${URL} -O install_anaconda.sh /bin/bash install_anaconda.sh -b -p ${CONDA} rm install_anaconda.sh -${CONDA}/bin/conda update -n base conda +ln -s ${CONDA}/etc/profile.d/conda.sh /etc/profile.d/conda.sh +chmod +x /usr/local/anaconda3/etc/profile.d/conda.sh + +conda update -n base conda +conda config --set channel_priority strict +conda config --prepend channels conda-forge +conda create -y --name ${CONDA_ENV_NAME} python=2 + popd diff --git a/docker/nexus-webapp/mesos/agent/docker-entrypoint.sh b/docker/nexus-webapp/mesos/agent/docker-entrypoint.sh index 3ce4313..c2013ab 100755 --- a/docker/nexus-webapp/mesos/agent/docker-entrypoint.sh +++ b/docker/nexus-webapp/mesos/agent/docker-entrypoint.sh @@ -17,6 +17,8 @@ set -ebx +source activate ${CONDA_ENV_NAME} + if [ -n "$TORNADO_HOST" ]; then sed -i "s/server.socket_host.*/server.socket_host=${TORNADO_HOST}/g" ${NEXUS_SRC}/analysis/webservice/config/web.ini fi @@ -39,4 +41,4 @@ cd ${NEXUS_SRC}/analysis python setup.py install --force # Set PROJ_LIB env var as workaround for missing environment variable for basemap https://github.com/conda-forge/basemap-feedstock/issues/30 -${MESOS_HOME}/build/bin/mesos-agent.sh --no-systemd_enable_support --launcher=posix --no-switch_user --executor_environment_variables='{"PYTHON_EGG_CACHE": "/tmp", "PROJ_LIB":"/usr/local/anaconda2/share/proj"}' "$@" +${MESOS_HOME}/build/bin/mesos-agent.sh --no-systemd_enable_support --launcher=posix --no-switch_user --executor_environment_variables="{\"PYTHON_EGG_CACHE\": \"/tmp\", \"PROJ_LIB\": \"${CONDA_HOME}/envs/${CONDA_ENV_NAME}/share/proj\"}" "$@" diff --git a/docker/nexus-webapp/mesos/base/Dockerfile b/docker/nexus-webapp/mesos/base/Dockerfile index c62a986..0442eb4 100644 --- a/docker/nexus-webapp/mesos/base/Dockerfile +++ b/docker/nexus-webapp/mesos/base/Dockerfile @@ -46,7 +46,8 @@ ENV INSTALL_LOC=/usr/local \ JAVA_HOME=/usr/java/default \ NEXUS_SRC=/tmp/incubator-sdap-nexus -ENV CONDA_HOME=${INSTALL_LOC}/anaconda2 \ +ENV CONDA_HOME=${INSTALL_LOC}/anaconda3 \ + CONDA_ENV_NAME=nexus-webapp \ MESOS_HOME=${INSTALL_LOC}/mesos-${MESOS_VER} \ SPARK_DIR=spark-${SPARK_VERSION} \ SPARK_PACKAGE=spark-${SPARK_VERSION}-bin-hadoop2.7 \ @@ -54,11 +55,9 @@ ENV CONDA_HOME=${INSTALL_LOC}/anaconda2 \ MESOS_PACKAGE=mesos-${MESOS_VER}.tar.gz ENV SPARK_HOME=${INSTALL_LOC}/${SPARK_DIR} \ - PYSPARK_DRIVER_PYTHON=${CONDA_HOME}/bin/python \ - PYSPARK_PYTHON=${CONDA_HOME}/bin/python \ - PYSPARK_SUBMIT_ARGS="--driver-memory=4g pyspark-shell" \ - # Workaround for missing environment variable for basemap https://github.com/conda-forge/basemap-feedstock/issues/30 - PROJ_LIB=${CONDA_HOME}/share/proj + PYSPARK_DRIVER_PYTHON=${CONDA_HOME}/envs/${CONDA_ENV_NAME}/bin/python \ + PYSPARK_PYTHON=${CONDA_HOME}/envs/${CONDA_ENV_NAME}/bin/python \ + PYSPARK_SUBMIT_ARGS="--driver-memory=4g pyspark-shell" ENV PYTHONPATH=${PYTHONPATH}:${SPARK_HOME}/python:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip:${SPARK_HOME}/python/lib/pyspark.zip \ MESOS_NATIVE_JAVA_LIBRARY=${INSTALL_LOC}/lib/libmesos.so \ @@ -79,10 +78,10 @@ RUN ./install_spark.sh "http://d3kbcqa49mib13.cloudfront.net"; ${SPARK_VERSION} $ # Install Anaconda COPY install_conda.sh ./install_conda.sh -RUN ./install_conda.sh "https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh"; ${CONDA_HOME} +RUN ./install_conda.sh "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh"; ${CONDA} # Conda dependencies for nexus -RUN conda config --add channels conda-forge && \ +RUN source activate ${CONDA_ENV_NAME} && \ conda install -y netCDF4 \ backports.functools_lru_cache \ numpy \ @@ -102,7 +101,8 @@ ARG APACHE_NEXUS_BRANCH=master ARG REBUILD_CODE=1 COPY install_nexusproto.sh ./install_nexusproto.sh COPY install_nexus.sh ./install_nexus.sh -RUN /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH && \ +RUN source activate ${CONDA_ENV_NAME} && \ + /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH && \ /tmp/install_nexus.sh $APACHE_NEXUS $APACHE_NEXUS_BRANCH $NEXUS_SRC CMD ["/bin/bash"] diff --git a/docker/nexus-webapp/mesos/webapp/docker-entrypoint.sh b/docker/nexus-webapp/mesos/webapp/docker-entrypoint.sh index e4d1abb..564f1bf 100755 --- a/docker/nexus-webapp/mesos/webapp/docker-entrypoint.sh +++ b/docker/nexus-webapp/mesos/webapp/docker-entrypoint.sh @@ -17,6 +17,8 @@ set -ebx +source activate ${CONDA_ENV_NAME} + if [ -n "$TORNADO_HOST" ]; then sed -i "s/server.socket_host.*/server.socket_host=${TORNADO_HOST}/g" ${NEXUS_SRC}/analysis/webservice/config/web.ini fi diff --git a/docker/nexus-webapp/standalone/Dockerfile b/docker/nexus-webapp/standalone/Dockerfile index 5570938..b474255 100644 --- a/docker/nexus-webapp/standalone/Dockerfile +++ b/docker/nexus-webapp/standalone/Dockerfile @@ -29,22 +29,23 @@ RUN yum -y update && \ yum clean all ARG SPARK_VERSION=2.2.0 +ENV CONDA=/usr/local/anaconda3 \ + CONDA_ENV_NAME=nexus-webapp ENV SPARK_LOCAL_IP=127.0.0.1 \ CASSANDRA_CONTACT_POINTS=127.0.0.1 \ + CASSANDRA_PORT=9042 \ CASSANDRA_LOCAL_DATACENTER=datacenter1 \ SOLR_URL_PORT=127.0.0.1:8983 \ SPARK_DIR=spark-${SPARK_VERSION} \ SPARK_PACKAGE=spark-${SPARK_VERSION}-bin-hadoop2.7 \ SPARK_HOME=/usr/local/spark-${SPARK_VERSION} \ - PYSPARK_DRIVER_PYTHON=/usr/local/anaconda2/bin/python \ - PYSPARK_PYTHON=/usr/local/anaconda2/bin/python \ + PYSPARK_DRIVER_PYTHON=${CONDA}/envs/${CONDA_ENV_NAME}/bin/python \ + PYSPARK_PYTHON=${CONDA}/envs/${CONDA_ENV_NAME}/bin/python \ PYSPARK_SUBMIT_ARGS="--driver-memory=4g pyspark-shell" \ PYTHONPATH=${PYTHONPATH}:/usr/local/spark-${SPARK_VERSION}/python:/usr/local/spark-${SPARK_VERSION}/python/lib/py4j-0.10.4-src.zip:/usr/local/spark-${SPARK_VERSION}/python/lib/pyspark.zip \ SPARK_EXECUTOR_URI=/usr/local/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz \ - NEXUS_SRC=/tmp/incubator-sdap-nexus \ - PATH=/usr/local/anaconda2/bin:$PATH \ - # Workaround for missing environment variable for basemap https://github.com/conda-forge/basemap-feedstock/issues/30 - PROJ_LIB=/usr/local/anaconda2/share/proj + NEXUS_SRC=/usr/local/incubator-sdap-nexus \ + PATH=${CONDA}/bin:${PATH} WORKDIR /tmp @@ -58,11 +59,10 @@ RUN ./install_spark.sh "http://d3kbcqa49mib13.cloudfront.net"; ${SPARK_VERSION} $ # Install Anaconda COPY install_conda.sh ./install_conda.sh -RUN ./install_conda.sh "https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh"; /usr/local/anaconda2 +RUN ./install_conda.sh "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh"; ${CONDA} # Conda dependencies for nexus -# RUN conda install -c conda-forge -y netCDF4 backports.functools_lru_cache=1.3 && \ -RUN conda config --add channels conda-forge && \ +RUN source activate ${CONDA_ENV_NAME} && \ conda install -y netCDF4 \ backports.functools_lru_cache \ numpy \ @@ -82,7 +82,8 @@ ARG APACHE_NEXUS_BRANCH=master ARG REBUILD_CODE=1 COPY install_nexusproto.sh ./install_nexusproto.sh COPY install_nexus.sh ./install_nexus.sh -RUN /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH && \ +RUN source activate ${CONDA_ENV_NAME} && \ + /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH && \ /tmp/install_nexus.sh $APACHE_NEXUS $APACHE_NEXUS_BRANCH $NEXUS_SRC COPY standalone/docker-entrypoint.sh ./docker-entrypoint.sh diff --git a/docker/nexus-webapp/standalone/docker-entrypoint.sh b/docker/nexus-webapp/standalone/docker-entrypoint.sh index 20270ac..adfcebc 100755 --- a/docker/nexus-webapp/standalone/docker-entrypoint.sh +++ b/docker/nexus-webapp/standalone/docker-entrypoint.sh @@ -17,6 +17,8 @@ set -ebx +source activate ${CONDA_ENV_NAME} + if [ -n "$TORNADO_HOST" ]; then sed -i "s/server.socket_host.*/server.socket_host=${TORNADO_HOST}/g" ${NEXUS_SRC}/analysis/webservice/config/web.ini fi
