This is an automated email from the ASF dual-hosted git repository. tloubrieu pushed a commit to branch moc_deployment in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit ed654f83188bb3430caedf0d71c01c0cea1a6a96 Author: thomas loubrieu <[email protected]> AuthorDate: Wed Jul 7 11:04:47 2021 -0700 make deployment run without root-containers, upgrade chart for spark 3.1.1 --- docker/nexus-webapp/Dockerfile | 66 +++++++++++------------------------------ helm/templates/_helpers.tpl | 3 ++ helm/templates/rolebinding.yaml | 5 ++-- helm/templates/webapp.yml | 6 ++-- 4 files changed, 27 insertions(+), 53 deletions(-) diff --git a/docker/nexus-webapp/Dockerfile b/docker/nexus-webapp/Dockerfile index 68f7d87..6a0cdcc 100644 --- a/docker/nexus-webapp/Dockerfile +++ b/docker/nexus-webapp/Dockerfile @@ -13,9 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM nexusjpl/alpine-pyspark:3.1.1 - - +#FROM nexusjpl/alpine-pyspark:3.1.1 +FROM tloubrieu/openshift-pyshark MAINTAINER Apache SDAP "[email protected]" @@ -23,53 +22,23 @@ ARG CONDA_VERSION="4.7.12.1" ARG CONDA_MD5="81c773ff87af5cfac79ab862942ab6b3" ARG CONDA_DIR="/opt/conda" -ENV \ - PYTHONPATH=/opt/conda/share/py4j/py4j0.10.9.2.jar \ - NEXUS_SRC=/tmp/incubator-sdap-nexus \ - PROJ_LIB=/opt/conda/lib/python3.8/site-packages/pyproj/data \ - PATH="$CONDA_DIR/bin:$PATH" \ - PYTHONDONTWRITEBYTECODE=1 \ - SPARK_HOME=/opt/spark \ - PYSPARK_DRIVER_PYTHON=/opt/conda/lib/python3.8 \ - PYSPARK_PYTHON=/opt/conda/lib/python3.8 \ - LD_LIBRARY_PATH=/usr/lib - -RUN apk add --update --no-cache \ - bzip2 \ - gcc \ - git \ - mesa-gl \ - wget \ - curl \ - which \ - python3 \ - bash==4.4.19-r1 \ - libc-dev \ - libressl2.7-libcrypto - -RUN apk upgrade musl +ENV PATH="/opt/conda/bin:${PATH}" +ARG PATH="/opt/conda/bin:${PATH}" + +RUN apt-get update && apt-get install -y wget && rm -rf /var/lib/apt/lists/* WORKDIR /tmp -RUN apk del libc6-compat -RUN apk --no-cache add wget zlib && \ - wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub && \ - wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.30-r0/glibc-2.30-r0.apk && \ - apk add glibc-2.30-r0.apk && \ - ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ - ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib && \ - ln -s /usr/lib/libgcc_s.so.1 /usr/glibc-compat/lib +RUN wget \ + https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && mkdir /root/.conda \ + && bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda \ + && rm -f Miniconda3-latest-Linux-x86_64.sh + + -COPY docker/nexus-webapp/install_conda.sh ./install_conda.sh -RUN /tmp/install_conda.sh RUN conda install python=3.8 -RUN cd /usr/lib && ln -s libcom_err.so.2 libcom_err.so.3 && \ - cd /opt/conda/lib && \ - ln -s libnetcdf.so.11 libnetcdf.so.7 && \ - ln -s libkea.so.1.4.6 libkea.so.1.4.5 && \ - ln -s libhdf5_cpp.so.12 libhdf5_cpp.so.10 && \ - ln -s libjpeg.so.9 libjpeg.so.8 # Change REBUILD_CODE if you want tell Docker not to use cached layers from this line on ARG REBUILD_CODE=0 @@ -77,6 +46,7 @@ ARG REBUILD_CODE=0 ARG APACHE_NEXUSPROTO=https://github.com/apache/incubator-sdap-nexusproto.git ARG APACHE_NEXUSPROTO_BRANCH=master +RUN apt-get update && apt-get install -y git COPY docker/nexus-webapp/install_nexusproto.sh ./install_nexusproto.sh RUN /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH @@ -97,8 +67,8 @@ RUN pip3 install -r requirements.txt RUN pip3 install cython RUN rm requirements.txt -WORKDIR /incubator-sdap-nexus +RUN chmod -R a+rwX /opt/conda/ -# Upgrade kubernetes client jar from the default version -RUN rm /opt/spark/jars/kubernetes-client-4.12.0.jar -ADD https://repo1.maven.org/maven2/io/fabric8/kubernetes-client/4.12.0/kubernetes-client-4.12.0.jar /opt/spark/jars +ENV PROJ_LIB=/opt/conda/lib/python3.8/site-packages/pyproj/data + +WORKDIR /incubator-sdap-nexus diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 3084fa9..f1cf360 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -35,6 +35,9 @@ The data volume which is used in both the Collection Manager and the Granule Ing {{- if .Values.ingestion.granules.path -}} hostPath: path: {{ .Values.ingestion.granules.path }} + {{- else }} + persistentVolumeClaim: + claimName: {{ .Values.ingestion.granules.pvc }} {{- end -}} {{- end -}} {{- end -}} diff --git a/helm/templates/rolebinding.yaml b/helm/templates/rolebinding.yaml index 34ee010..9366fa4 100644 --- a/helm/templates/rolebinding.yaml +++ b/helm/templates/rolebinding.yaml @@ -5,7 +5,8 @@ metadata: app.kubernetes.io/instance: spark-operator app.kubernetes.io/name: spark-operator app.kubernetes.io/version: v1beta2-1.2.3-3.1.1 - name: spark + name: sdap-spark-rolebinding + namespace: {{ .Release.Namespace }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -13,5 +14,5 @@ roleRef: subjects: - kind: ServiceAccount name: sdap-spark - namespace: $(namespace) + namespace: {{ .Release.Namespace }} diff --git a/helm/templates/webapp.yml b/helm/templates/webapp.yml index 88a024a..f921407 100644 --- a/helm/templates/webapp.yml +++ b/helm/templates/webapp.yml @@ -16,7 +16,7 @@ spec: - --cassandra-username={{ include "nexus.credentials.cassandra.username" . }} - --cassandra-password={{ include "nexus.credentials.cassandra.password" . }} - --solr-host={{ include "nexus.urls.solr" . }} - sparkVersion: "2.4.4" + sparkVersion: "3.1.1" restartPolicy: type: OnFailure onFailureRetries: 10 @@ -26,12 +26,12 @@ spec: driver: {{ .Values.webapp.distributed.driver | toYaml | indent 4 }} labels: - version: 2.4.4 + version: 3.1.1 serviceAccount: sdap-spark executor: {{ .Values.webapp.distributed.executor| toYaml | indent 4 }} labels: - version: 2.4.4 + version: 3.1.1 ---
