Repository: incubator-singa Updated Branches: refs/heads/master 330c87920 -> f94ec89fc
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f94ec89f/doc/notebook/utils.py ---------------------------------------------------------------------- diff --git a/doc/notebook/utils.py b/doc/notebook/utils.py deleted file mode 100755 index ff772ad..0000000 --- a/doc/notebook/utils.py +++ /dev/null @@ -1,138 +0,0 @@ -""" This file contains different utility functions that are not connected -in anyway to the networks presented in the tutorials, but rather help in -processing the outputs into a more understandable way. - -For example ``tile_raster_images`` helps in generating a easy to grasp -image from a set of samples or weights. -""" - -import numpy - - -def scale_to_unit_interval(ndar, eps=1e-8): - """ Scales all values in the ndarray ndar to be between 0 and 1 """ - ndar = ndar.copy() - ndar -= ndar.min() - ndar *= 1.0 / (ndar.max() + eps) - return ndar - - -def tile_raster_images(X, img_shape, tile_shape, tile_spacing=(0, 0), - scale_rows_to_unit_interval=True, - output_pixel_vals=True): - """ - Transform an array with one flattened image per row, into an array in - which images are reshaped and layed out like tiles on a floor. - - This function is useful for visualizing datasets whose rows are images, - and also columns of matrices for transforming those rows - (such as the first layer of a neural net). - - :type X: a 2-D ndarray or a tuple of 4 channels, elements of which can - be 2-D ndarrays or None; - :param X: a 2-D array in which every row is a flattened image. - - :type img_shape: tuple; (height, width) - :param img_shape: the original shape of each image - - :type tile_shape: tuple; (rows, cols) - :param tile_shape: the number of images to tile (rows, cols) - - :param output_pixel_vals: if output should be pixel values (i.e. int8 - values) or floats - - :param scale_rows_to_unit_interval: if the values need to be scaled before - being plotted to [0,1] or not - - - :returns: array suitable for viewing as an image. - (See:`Image.fromarray`.) - :rtype: a 2-d array with same dtype as X. - - """ - - assert len(img_shape) == 2 - assert len(tile_shape) == 2 - assert len(tile_spacing) == 2 - - # The expression below can be re-written in a more C style as - # follows : - # - # out_shape = [0,0] - # out_shape[0] = (img_shape[0]+tile_spacing[0])*tile_shape[0] - - # tile_spacing[0] - # out_shape[1] = (img_shape[1]+tile_spacing[1])*tile_shape[1] - - # tile_spacing[1] - out_shape = [ - (ishp + tsp) * tshp - tsp - for ishp, tshp, tsp in zip(img_shape, tile_shape, tile_spacing) - ] - - if isinstance(X, tuple): - assert len(X) == 4 - # Create an output numpy ndarray to store the image - if output_pixel_vals: - out_array = numpy.zeros((out_shape[0], out_shape[1], 4), - dtype='uint8') - else: - out_array = numpy.zeros((out_shape[0], out_shape[1], 4), - dtype=X.dtype) - - #colors default to 0, alpha defaults to 1 (opaque) - if output_pixel_vals: - channel_defaults = [0, 0, 0, 255] - else: - channel_defaults = [0., 0., 0., 1.] - - for i in range(4): - if X[i] is None: - # if channel is None, fill it with zeros of the correct - # dtype - dt = out_array.dtype - if output_pixel_vals: - dt = 'uint8' - out_array[:, :, i] = numpy.zeros( - out_shape, - dtype=dt - ) + channel_defaults[i] - else: - # use a recurrent call to compute the channel and store it - # in the output - out_array[:, :, i] = tile_raster_images( - X[i], img_shape, tile_shape, tile_spacing, - scale_rows_to_unit_interval, output_pixel_vals) - return out_array - - else: - # if we are dealing with only one channel - H, W = img_shape - Hs, Ws = tile_spacing - - # generate a matrix to store the output - dt = X.dtype - if output_pixel_vals: - dt = 'uint8' - out_array = numpy.zeros(out_shape, dtype=dt) - - for tile_row in range(tile_shape[0]): - for tile_col in range(tile_shape[1]): - if tile_row * tile_shape[1] + tile_col < X.shape[0]: - this_x = X[tile_row * tile_shape[1] + tile_col] - if scale_rows_to_unit_interval: - # if we should scale values to be between 0 and 1 - # do this by calling the `scale_to_unit_interval` - # function - this_img = scale_to_unit_interval( - this_x.reshape(img_shape)) - else: - this_img = this_x.reshape(img_shape) - # add the slice to the corresponding position in the - # output array - c = 1 - if output_pixel_vals: - c = 255 - out_array[ - tile_row * (H + Hs): tile_row * (H + Hs) + H, - tile_col * (W + Ws): tile_col * (W + Ws) + W - ] = this_img * c - return out_array http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f94ec89f/tool/jenkins/README.md ---------------------------------------------------------------------- diff --git a/tool/jenkins/README.md b/tool/jenkins/README.md index b93b221..339e532 100644 --- a/tool/jenkins/README.md +++ b/tool/jenkins/README.md @@ -12,12 +12,12 @@ Those built binaries need to be archived for users to download. [Jenkins Official Wiki](https://wiki.jenkins-ci.org/display/JENKINS/Installing+Jenkins) The slave nodes for running different building environments are configured under 'Manage Jenkins'->'Manage nodes'. -## Configure Jenkins Multi-configuration Project +## Configure Jenkins Multi-configuration Project for Unit Testing and PySINGA Generation Create a multi-configuration project and configure project as follows: ### Description - This job automatically pulls latest commits from SINGA github repository. - It compiles and tests SINGA in different environments and creates PySINGA wheel distribution accordingly. +This job automatically pulls latest commits from apache SINGA github repository. +It compiles and tests SINGA in different environments and creates PySINGA wheel distribution accordingly. ### General * Discard old builds - Max # of builds to keep - 50 @@ -56,6 +56,42 @@ Create a multi-configuration project and configure project as follows: 32/84d56b7/ubuntu16.04-cuda8.0-cudnn5/singa-1.0.1-py2-none-any.whl +## Configure Jenkins for SINGA Website Updates + +### Description and Configuration + +This job is triggered upon any changes to the files of the `doc/` folder. +It does the following tasks, + +1. installs the latest PySINGA +2. pull the latest source code +3. generate the html files for the documentation +4. update the SINGA website + +The Jenkins job configuration is similar as above except the following fields, + +* Source Code Management - Git - Additional Behaviors - Include Region `doc/*` +* Build - Execute Shell - Command `bash -ex tool/jenkins/jenkins_doc.sh` +* No `Post-build Actions` + +### Docker Images + +The Docker image for the Jenkins slave node is at `docker/ubuntu16.04/runtime/Dockerfile`. +To build the docker image, + + # under the docker/ubuntu16.04/runtime/ folder + $ docker built -t singa:doc . + +To start the slave node + + $ docker run --name singa-doc -d singa:doc + $ docker exec -it singa-doc /bin/bash + $ svn co https://svn.apache.org/repos/asf/incubator/singa/site/trunk + # update ~/.subversion/config to set 'store-password=yes' + # to set password free commit, we have to do a manual commit at first. + # change any file (add spaces) inside trunk/ to commit a message + $ svn commit -m "test" --username <committer id> --password <passwd> + ## Docker Images We provide in `docker` a number of singa docker images for Jenkins to use as slaves. To run the docker images, http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f94ec89f/tool/jenkins/docker/ubuntu14.04/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/ubuntu14.04/Dockerfile b/tool/jenkins/docker/ubuntu14.04/Dockerfile deleted file mode 100644 index 838be4a..0000000 --- a/tool/jenkins/docker/ubuntu14.04/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -# Base unbuntu 14.04 image from nvidia/cuda -# Change tags to build with different cuda/cudnn versions: -# FROM nvidia/cuda:8.0-cudnn5-devel -# FROM nvidia/cuda:7.5-cudnn5-devel -# FROM nvidia/cuda:7.5-cudnn4-devel -# FROM nvidia/cuda:7.0-cudnn4-devel -FROM nvidia/cuda:7.5-cudnn5-devel - -# install dependencies -RUN apt-get update \ - && apt-get install -y --no-install-recommends git build-essential autoconf libtool cmake libpcre3-dev libprotobuf-dev libopenblas-dev protobuf-compiler python-dev python-pip wget openssh-server\ - && apt-get clean && apt-get autoremove && apt-get autoclean \ - && rm -rf /var/lib/apt/lists/* \ - && pip install -U pip wheel numpy setuptools unittest-xml-reporting protobuf - - -# install swig 3.0 -RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \ - tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \ - ./configure && make && make install - -# set environment -ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} -ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} -ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} -ENV OS_VERSION ubuntu14.04 - -# download singa source -RUN git clone https://github.com/apache/incubator-singa.git - -# config ssh service -RUN mkdir /var/run/sshd -RUN echo 'root:singa' | chpasswd -RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config -# SSH login fix. Otherwise user is kicked off after login -RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -# dump environment variables into files, so that ssh can see also -RUN env | grep _ >> /etc/environment - -EXPOSE 22 - -CMD ["/usr/sbin/sshd", "-D"] http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f94ec89f/tool/jenkins/docker/ubuntu14.04/devel/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/ubuntu14.04/devel/Dockerfile b/tool/jenkins/docker/ubuntu14.04/devel/Dockerfile new file mode 100644 index 0000000..838be4a --- /dev/null +++ b/tool/jenkins/docker/ubuntu14.04/devel/Dockerfile @@ -0,0 +1,43 @@ +# Base unbuntu 14.04 image from nvidia/cuda +# Change tags to build with different cuda/cudnn versions: +# FROM nvidia/cuda:8.0-cudnn5-devel +# FROM nvidia/cuda:7.5-cudnn5-devel +# FROM nvidia/cuda:7.5-cudnn4-devel +# FROM nvidia/cuda:7.0-cudnn4-devel +FROM nvidia/cuda:7.5-cudnn5-devel + +# install dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends git build-essential autoconf libtool cmake libpcre3-dev libprotobuf-dev libopenblas-dev protobuf-compiler python-dev python-pip wget openssh-server\ + && apt-get clean && apt-get autoremove && apt-get autoclean \ + && rm -rf /var/lib/apt/lists/* \ + && pip install -U pip wheel numpy setuptools unittest-xml-reporting protobuf + + +# install swig 3.0 +RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \ + tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \ + ./configure && make && make install + +# set environment +ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} +ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} +ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} +ENV OS_VERSION ubuntu14.04 + +# download singa source +RUN git clone https://github.com/apache/incubator-singa.git + +# config ssh service +RUN mkdir /var/run/sshd +RUN echo 'root:singa' | chpasswd +RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# SSH login fix. Otherwise user is kicked off after login +RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +# dump environment variables into files, so that ssh can see also +RUN env | grep _ >> /etc/environment + +EXPOSE 22 + +CMD ["/usr/sbin/sshd", "-D"] http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f94ec89f/tool/jenkins/docker/ubuntu16.04/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/ubuntu16.04/Dockerfile b/tool/jenkins/docker/ubuntu16.04/Dockerfile deleted file mode 100644 index da8836e..0000000 --- a/tool/jenkins/docker/ubuntu16.04/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -# Base unbuntu 16.04 image from nvidia/cuda -# Change tags to build with different cuda/cudnn versions: -# FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04 -FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04 - -# install dependencies - -MAINTAINER incubator-singa [email protected] - -# install dependencies -RUN apt-get update \ - && apt-get install -y --no-install-recommends git g++ cmake libprotobuf-dev libopenblas-dev protobuf-compiler python-dev python-pip swig wget openssh-server\ - && apt-get clean && apt-get autoremove && apt-get autoclean \ - && rm -rf /var/lib/apt/lists/* \ - && pip --no-cache-dir install -U pip wheel numpy setuptools unittest-xml-reporting protobuf - -# set environment -ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} -ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} -ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} -ENV OS_VERSION ubuntu16.04 - -# download singa source -RUN git clone https://github.com/apache/incubator-singa.git - -# config ssh service -RUN mkdir /var/run/sshd -RUN echo 'root:singa' | chpasswd -RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config -# SSH login fix. Otherwise user is kicked off after login -RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -# dump environment variables into files, so that ssh can see also -RUN env | grep _ >> /etc/environment - -EXPOSE 22 - -CMD ["/usr/sbin/sshd", "-D"] http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f94ec89f/tool/jenkins/docker/ubuntu16.04/devel/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/ubuntu16.04/devel/Dockerfile b/tool/jenkins/docker/ubuntu16.04/devel/Dockerfile new file mode 100644 index 0000000..da8836e --- /dev/null +++ b/tool/jenkins/docker/ubuntu16.04/devel/Dockerfile @@ -0,0 +1,38 @@ +# Base unbuntu 16.04 image from nvidia/cuda +# Change tags to build with different cuda/cudnn versions: +# FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04 +FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04 + +# install dependencies + +MAINTAINER incubator-singa [email protected] + +# install dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends git g++ cmake libprotobuf-dev libopenblas-dev protobuf-compiler python-dev python-pip swig wget openssh-server\ + && apt-get clean && apt-get autoremove && apt-get autoclean \ + && rm -rf /var/lib/apt/lists/* \ + && pip --no-cache-dir install -U pip wheel numpy setuptools unittest-xml-reporting protobuf + +# set environment +ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH} +ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH} +ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH} +ENV OS_VERSION ubuntu16.04 + +# download singa source +RUN git clone https://github.com/apache/incubator-singa.git + +# config ssh service +RUN mkdir /var/run/sshd +RUN echo 'root:singa' | chpasswd +RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# SSH login fix. Otherwise user is kicked off after login +RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +# dump environment variables into files, so that ssh can see also +RUN env | grep _ >> /etc/environment + +EXPOSE 22 + +CMD ["/usr/sbin/sshd", "-D"] http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f94ec89f/tool/jenkins/docker/ubuntu16.04/runtime/Dockerfile ---------------------------------------------------------------------- diff --git a/tool/jenkins/docker/ubuntu16.04/runtime/Dockerfile b/tool/jenkins/docker/ubuntu16.04/runtime/Dockerfile new file mode 100644 index 0000000..2aeea94 --- /dev/null +++ b/tool/jenkins/docker/ubuntu16.04/runtime/Dockerfile @@ -0,0 +1,25 @@ +# Base unbuntu 16.04 image +FROM ubuntu:latest + +MAINTAINER incubator-singa [email protected] + +# install dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends subversion git python-dev python-pip python-sphinx openssh-server\ + && apt-get clean && apt-get autoremove && apt-get autoclean \ + && rm -rf /var/lib/apt/lists/* \ + && pip --no-cache-dir install -U pip setuptools \ + && pip --no-cache-dir install recommonmark + +RUN mkdir /var/run/sshd +RUN echo 'root:singa' | chpasswd +RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config +# SSH login fix. Otherwise user is kicked off after login +RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +# dump environment variables into files, so that ssh can see also +# RUN env | grep _ >> /etc/environment + +EXPOSE 22 + +CMD ["/usr/sbin/sshd", "-D"] http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f94ec89f/tool/jenkins/jenkins_doc.sh ---------------------------------------------------------------------- diff --git a/tool/jenkins/jenkins_doc.sh b/tool/jenkins/jenkins_doc.sh new file mode 100644 index 0000000..2bb2333 --- /dev/null +++ b/tool/jenkins/jenkins_doc.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +#/** +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# This script is used by Jenkins to update SINGA website + +echo Install PySINGA, generate HTML files and update SINGA website +pip install --upgrade http://www.comp.nus.edu.sg/~dbsystem/singa/assets/file/whl/latest/ubuntu16.04-cpp/singa-1.0.1-py2-none-any.whl +COMMIT=`git rev-parse --short HEAD` +cd doc +# generate the html files +./build.sh html +# checkout the current website files +svn co https://svn.apache.org/repos/asf/incubator/singa/site/trunk +# overwrite the existing files +cp -r _build/html/* trunk/ +# track newly added files and commit +cd trunk +svn add --force * --auto-props --parents --depth infinity -q +svn commit -m "update the docs by jenkins for commit $COMMIT"
