This is an automated email from the ASF dual-hosted git repository.
shinsuke pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/predictionio.git
The following commit(s) were added to refs/heads/develop by this push:
new 4342fcd [PIO-183] Add Jupyter Docker image (#481)
4342fcd is described below
commit 4342fcd9d0a7b549543b59467f5e1b008523fe4f
Author: Shinsuke Sugaya <[email protected]>
AuthorDate: Fri Nov 2 12:31:42 2018 +0900
[PIO-183] Add Jupyter Docker image (#481)
---
docker/.ivy2/.keep | 14 +++
docker/JUPYTER.md | 166 ++++++++++++++++++++++++++++++
docker/README.md | 26 ++++-
docker/docker-compose.jupyter.yml | 31 ++++++
docker/jupyter/Dockerfile | 100 ++++++++++++++++++
docker/jupyter/fix-permissions | 36 +++++++
docker/jupyter/jupyter_notebook_config.py | 18 ++++
docker/jupyter/requirements.txt | 14 +++
docker/jupyter/start-jupyter.sh | 37 +++++++
docker/jupyter/start.sh | 81 +++++++++++++++
docker/pio/Dockerfile | 2 +
11 files changed, 522 insertions(+), 3 deletions(-)
diff --git a/docker/.ivy2/.keep b/docker/.ivy2/.keep
new file mode 100644
index 0000000..ec20143
--- /dev/null
+++ b/docker/.ivy2/.keep
@@ -0,0 +1,14 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/docker/JUPYTER.md b/docker/JUPYTER.md
new file mode 100644
index 0000000..7a6e988
--- /dev/null
+++ b/docker/JUPYTER.md
@@ -0,0 +1,166 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+Jupyter With PredictionIO
+=========================
+
+## Overview
+
+Using Jupyter based docker, you can use Jupyter Notebook with PredictionIO
environment.
+It helps you with your exploratory data analysis (EDA).
+
+## Run Jupyter Notebook
+
+First of all, start Jupyter container with PredictionIO environment:
+
+```
+docker-compose -f docker-compose.jupyter.yml \
+ -f pgsql/docker-compose.base.yml \
+ -f pgsql/docker-compose.meta.yml \
+ -f pgsql/docker-compose.event.yml \
+ -f pgsql/docker-compose.model.yml \
+ up
+```
+
+The above command prints a token to the console as below.
+
+```
+pio_1 | http://(3aaf67361022 or
127.0.0.1):8888/?token=e87a634b4ab7e2c8bcd86aea9def3eb48183c043eac86f3e
+```
+
+Open `http://127.0.0.1:8888/`, type the token, and then open a new terminal in
Jupyter from `New` pulldown button.
+
+## Getting Started With Scala Based Template
+
+### Download Template
+
+Clone a template using Git:
+
+```
+cd templates/
+git clone https://github.com/apache/predictionio-template-recommender.git
+cd predictionio-template-recommender/
+```
+
+Replace a name with `MyApp1`.
+
+```
+sed -i "s/INVALID_APP_NAME/MyApp1/" engine.json
+```
+
+### Register New Application
+
+Using pio command, register a new application as `MyApp1`.
+
+```
+pio app new MyApp1
+```
+
+This command prints an access key as below.
+
+```
+[INFO] [Pio$] Access Key:
bbe8xRHN1j3Sa8WeAT8TSxt5op3lUqhvXmKY1gLRjg70K-DUhHIJJ0-UzgKumxGm
+```
+
+Set it to an environment variable `ACCESS_KEY`.
+
+```
+ACCESS_KEY=bbe8xRHN1j3Sa8WeAT8TSxt5op3lUqhvXmKY1gLRjg70K-DUhHIJJ0-UzgKumxGm
+```
+
+### Import Training Data
+
+Download trainging data and import them to PredictionIO Event server.
+
+```
+curl
https://raw.githubusercontent.com/apache/spark/master/data/mllib/sample_movielens_data.txt
--create-dirs -o data/sample_movielens_data.txt
+python data/import_eventserver.py --access_key $ACCESS_KEY
+```
+
+### Build Template
+
+Build your template by the following command:
+
+```
+pio build --verbose
+```
+
+### Create Model
+
+To create a model, run:
+
+```
+pio train
+```
+
+## Getting Started With Python Based Template
+
+### Download Template
+
+Clone a template using Git:
+
+```
+cd templates/
+git clone https://github.com/jpioug/predictionio-template-iris.git
+predictionio-template-iris/
+```
+
+### Register New Application
+
+Using pio command, register a new application as `IrisApp`.
+
+```
+pio app new --access-key IRIS_TOKEN IrisApp
+```
+
+### Import Training Data
+
+Download trainging data and import them to PredictionIO Event server.
+
+```
+python data/import_eventserver.py
+```
+
+### Build Template
+
+Build your template by the following command:
+
+```
+pio build --verbose
+```
+
+### EDA
+
+To do data analysis, open `templates/predictionio-template-iris/eda.ipynb` on
Jupyter.
+
+### Create Model
+
+You need to clear the following environment variables in the terminal before
executing `pio train`.
+
+```
+unset PYSPARK_PYTHON
+unset PYSPARK_DRIVER_PYTHON
+unset PYSPARK_DRIVER_PYTHON_OPTS
+```
+
+To create a model, run:
+
+```
+pio train --main-py-file train.py
+```
+
+
diff --git a/docker/README.md b/docker/README.md
index f1bc5a0..97e1bf5 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -231,19 +231,39 @@ docker-compose -f docker-compose.yml \
See `deploy/run.sh` and `docker-compose.deploy.yml` if changing a deployment.
+### Run with Jupyter
+
+You can launch PredictionIO with Jupyter.
+
+```
+docker-compose -f docker-compose.jupyter.yml \
+ -f pgsql/docker-compose.base.yml \
+ -f pgsql/docker-compose.meta.yml \
+ -f pgsql/docker-compose.event.yml \
+ -f pgsql/docker-compose.model.yml \
+ up
+```
+
+For more information, see [JUPYTER.md](./JUPYTER.md).
## Development
-### Build Docker Image
+### Build Base Docker Image
```
docker build -t predictionio/pio pio
```
+### Build Jupyter Docker Image
+
+```
+docker build -t predictionio/pio-jupyter jupyter
+```
+
### Push Docker Image
```
docker push predictionio/pio:latest
-docker tag predictionio/pio:latest predictionio/pio:$PIO\_VERSION
-docker push predictionio/pio:$PIO\_VERSION
+docker tag predictionio/pio:latest predictionio/pio:$PIO_VERSION
+docker push predictionio/pio:$PIO_VERSION
```
diff --git a/docker/docker-compose.jupyter.yml
b/docker/docker-compose.jupyter.yml
new file mode 100644
index 0000000..e2667bb
--- /dev/null
+++ b/docker/docker-compose.jupyter.yml
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ pio:
+ image: predictionio/pio-jupyter:latest
+ ports:
+ - 7070:7070
+ - 8000:8000
+ - 8888:8888
+ volumes:
+ - ./templates:/home/jovyan/templates
+ - ./.ivy2:/home/jovyan/.ivy2
+ environment:
+ - CHOWN_HOME=yes
+ - GRANT_SUDO=yes
+ - VOLUME_UID=yes
+ dns: 8.8.8.8
diff --git a/docker/jupyter/Dockerfile b/docker/jupyter/Dockerfile
new file mode 100644
index 0000000..78e71b2
--- /dev/null
+++ b/docker/jupyter/Dockerfile
@@ -0,0 +1,100 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM predictionio/pio:latest
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update \
+ && apt install -y build-essential curl git gcc make openssl libssl-dev
libbz2-dev \
+ apt-transport-https ca-certificates g++ gnupg graphviz lsb-release
openssh-client zip \
+ libreadline-dev libsqlite3-dev cmake libxml2-dev wget bzip2 sudo vim unzip
locales \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen
+
+ENV LC_ALL=en_US.UTF-8 \
+ LANG=en_US.UTF-8 \
+ LANGUAGE=en_US.UTF-8 \
+ NB_USER=jovyan \
+ NB_UID=1000 \
+ NB_GID=100 \
+ CONDA_DIR=/opt/conda \
+ PIP_DEFAULT_TIMEOUT=180
+ENV PATH=$CONDA_DIR/bin:$PATH \
+ HOME=/home/$NB_USER
+
+ADD fix-permissions /usr/local/bin/fix-permissions
+RUN chmod +x /usr/local/bin/fix-permissions \
+ && groupadd wheel -g 11 \
+ && echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su \
+ && useradd -m -s /bin/bash -N -u $NB_UID $NB_USER \
+ && mkdir -p $CONDA_DIR \
+ && chmod g+w /etc/passwd \
+ && fix-permissions $HOME \
+ && fix-permissions $CONDA_DIR
+
+USER $NB_USER
+
+ENV MINICONDA_VERSION 4.4.10
+RUN wget -q
https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh
-O /tmp/miniconda.sh \
+ && echo 'bec6203dbb2f53011e974e9bf4d46e93 */tmp/miniconda.sh' | md5sum -c
- \
+ && bash /tmp/miniconda.sh -f -b -p $CONDA_DIR \
+ && rm /tmp/miniconda.sh \
+ && conda config --system --prepend channels conda-forge \
+ && conda config --system --set auto_update_conda false \
+ && conda config --system --set show_channel_urls true \
+ && conda install --quiet --yes conda="${MINICONDA_VERSION%.*}.*" \
+ && conda update --all --quiet --yes \
+ && conda clean -tipsy \
+ && rm -rf /home/$NB_USER/.cache/yarn \
+ && fix-permissions $CONDA_DIR \
+ && fix-permissions /home/$NB_USER
+
+RUN conda install --quiet --yes 'tini=0.18.0' \
+ && conda list tini | grep tini | tr -s ' ' | cut -d ' ' -f 1,2 >>
$CONDA_DIR/conda-meta/pinned \
+ && conda clean -tipsy \
+ && fix-permissions $CONDA_DIR \
+ && fix-permissions /home/$NB_USER
+
+RUN conda install --quiet --yes 'notebook=5.6.*' 'jupyterlab=0.34.*' nodejs\
+ && jupyter labextension install @jupyterlab/hub-extension@^0.11.0 \
+ && jupyter notebook --generate-config \
+ && conda clean -tipsy \
+ && npm cache clean --force \
+ && rm -rf $CONDA_DIR/share/jupyter/lab/staging \
+ && rm -rf /home/$NB_USER/.cache/yarn \
+ && fix-permissions $CONDA_DIR \
+ && fix-permissions /home/$NB_USER
+
+ADD requirements.txt /tmp/requirements.txt
+RUN pip --no-cache-dir install -r /tmp/requirements.txt \
+ && fix-permissions $CONDA_DIR \
+ && fix-permissions /home/$NB_USER
+
+COPY jupyter_notebook_config.py /home/$NB_USER/.jupyter/
+COPY start*.sh /usr/local/bin/
+
+USER root
+RUN chmod +x /usr/local/bin/*.sh
+
+EXPOSE 8888
+WORKDIR $HOME
+ENTRYPOINT ["tini", "--"]
+CMD ["/usr/local/bin/start-jupyter.sh"]
+
diff --git a/docker/jupyter/fix-permissions b/docker/jupyter/fix-permissions
new file mode 100644
index 0000000..ab276ec
--- /dev/null
+++ b/docker/jupyter/fix-permissions
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+for d in $@; do
+ find "$d" \
+ ! \( \
+ -group $NB_GID \
+ -a -perm -g+rwX \
+ \) \
+ -exec chgrp $NB_GID {} \; \
+ -exec chmod g+rwX {} \;
+ find "$d" \
+ \( \
+ -type d \
+ -a ! -perm -6000 \
+ \) \
+ -exec chmod +6000 {} \;
+done
diff --git a/docker/jupyter/jupyter_notebook_config.py
b/docker/jupyter/jupyter_notebook_config.py
new file mode 100644
index 0000000..1bd9f2a
--- /dev/null
+++ b/docker/jupyter/jupyter_notebook_config.py
@@ -0,0 +1,18 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+c = get_config()
+c.NotebookApp.ip = '*'
+c.NotebookApp.port = 8888
+c.NotebookApp.open_browser = False
+
diff --git a/docker/jupyter/requirements.txt b/docker/jupyter/requirements.txt
new file mode 100644
index 0000000..b8a5c32
--- /dev/null
+++ b/docker/jupyter/requirements.txt
@@ -0,0 +1,14 @@
+cython
+google-cloud
+h5py
+ipywidgets
+jupyter_contrib_nbextensions
+keras
+matplotlib
+pandas
+pandas-gbq
+predictionio
+sklearn
+tensor2tensor
+tensorflow
+widgetsnbextension
diff --git a/docker/jupyter/start-jupyter.sh b/docker/jupyter/start-jupyter.sh
new file mode 100644
index 0000000..1458e54
--- /dev/null
+++ b/docker/jupyter/start-jupyter.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+# store PIO environment to pio-env.sh
+PIO_ENV_FILE=/etc/predictionio/pio-env.sh
+env | grep ^PIO_ >> $PIO_ENV_FILE
+if [ $(grep _MYSQL_ $PIO_ENV_FILE | wc -l) = 0 ] ; then
+ sed -i "s/^MYSQL/#MYSQL/" $PIO_ENV_FILE
+fi
+
+# start event server
+sh /usr/bin/pio_run &
+
+export PYSPARK_PYTHON=$CONDA_DIR/bin/python
+export PYSPARK_DRIVER_PYTHON=$CONDA_DIR/bin/jupyter
+export PYSPARK_DRIVER_PYTHON_OPTS=notebook
+
+. /usr/local/bin/start.sh $PIO_HOME/bin/pio-shell --with-pyspark
+
diff --git a/docker/jupyter/start.sh b/docker/jupyter/start.sh
new file mode 100644
index 0000000..c4b9330
--- /dev/null
+++ b/docker/jupyter/start.sh
@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+if [[ "$VOLUME_UID" == "1" || "$VOLUME_UID" == 'yes' ]]; then
+ DIR_UID=`ls -lnd /home/jovyan/templates | awk '{print $3}'`
+ if [ x"$DIR_UID" != "x" -a x"$DIR_UID" != "x0" ] ; then
+ NB_UID=$DIR_UID
+ fi
+fi
+
+if [ $(id -u) == 0 ] ; then
+ if id jovyan &> /dev/null ; then
+ echo "Set username to $NB_USER"
+ usermod -d /home/$NB_USER -l $NB_USER jovyan
+ fi
+
+ if [[ "$CHOWN_HOME" == "1" || "$CHOWN_HOME" == 'yes' ]]; then
+ echo "Change ownership of /home/$NB_USER to $NB_UID"
+ chown -R $NB_UID /home/$NB_USER
+ fi
+ if [ ! -z "$CHOWN_EXTRA" ]; then
+ for extra_dir in $(echo $CHOWN_EXTRA | tr ',' ' '); do
+ chown -R $NB_UID $extra_dir
+ done
+ fi
+
+ if [[ "$NB_USER" != "jovyan" ]]; then
+ if [[ ! -e "/home/$NB_USER" ]]; then
+ echo "Move home dir to /home/$NB_USER"
+ mv /home/jovyan "/home/$NB_USER"
+ fi
+ if [[ "$PWD/" == "/home/jovyan/"* ]]; then
+ newcwd="/home/$NB_USER/${PWD:13}"
+ echo "Set CWD to $newcwd"
+ cd "$newcwd"
+ fi
+ fi
+
+ if [ "$NB_UID" != $(id -u $NB_USER) ] ; then
+ echo "Set $NB_USER to uid:$NB_UID"
+ usermod -u $NB_UID $NB_USER
+ fi
+
+ if [ "$NB_GID" != $(id -g $NB_USER) ] ; then
+ echo "Add $NB_USER to gid:$NB_GID"
+ groupadd -g $NB_GID -o ${NB_GROUP:-${NB_USER}}
+ usermod -g $NB_GID -a -G $NB_GID,100 $NB_USER
+ fi
+
+ if [[ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == 'yes' ]]; then
+ echo "Set sudo access to $NB_USER"
+ echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook
+ fi
+
+ echo "Execute command as $NB_USER"
+ exec su $NB_USER -c "env PATH=$PATH $*"
+
+else
+ echo "Execute command"
+ exec $*
+
+fi
+
diff --git a/docker/pio/Dockerfile b/docker/pio/Dockerfile
index 88930bd..a63183e 100644
--- a/docker/pio/Dockerfile
+++ b/docker/pio/Dockerfile
@@ -42,6 +42,8 @@ RUN bash ./make-distribution.sh \
-Delasticsearch.version=$ELASTICSEARCH_VERSION \
--with-deb && \
dpkg -i ./assembly/target/predictionio_*.deb && \
+ cp -r ./python /usr/share/predictionio && \
+ mkdir /var/log/predictionio && \
rm -rf /opt/src/predictionio/*
