This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 5d0d7e7 SUBMARINE-533. Add pysubmarine ci dockerfile
5d0d7e7 is described below
commit 5d0d7e76579a63f2a512b3572913ce8549be9009
Author: pingsutw <[email protected]>
AuthorDate: Tue Jul 7 06:31:15 2020 +0800
SUBMARINE-533. Add pysubmarine ci dockerfile
### What is this PR for?
Add pysubmarine ci docker image
The reason to add this dockerfile
1. The main reason is to generate python API from swagger without install
any dependencies (java, python, code-jen.jar..etc)
2. Run pysubmarine testing and formatting Code inside docker
3. We could also build submarine inside docker
4. pysubmarine development
### What type of PR is it?
[Improvement]
### Todos
* [ ] - Task
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-533
### How should this be tested?
https://travis-ci.org/github/pingsutw/hadoop-submarine/builds/706069697
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: pingsutw <[email protected]>
Closes #343 from pingsutw/SUBMARINE-533 and squashes the following commits:
9963791 [pingsutw] Fix bug
5a38e7f [pingsutw] Add bootstrap.sh
73a75bf [pingsutw] Update
7c6a631 [pingsutw] SUBMARINE-533 Add pysubmarine ci dockerfile
---
dev-support/pysubmarine/Dockerfile | 53 ++++++++++++++++++++++
dev-support/pysubmarine/bootstrap.sh | 36 +++++++++++++++
.../pysubmarine/install-conda.sh | 43 +++++++++---------
dev-support/pysubmarine/run-pysubmarine-ci.sh | 40 ++++++++++++++++
docs/submarine-sdk/pysubmarine/development.md | 10 ++++
submarine-sdk/pysubmarine/github-actions/lint.sh | 5 +-
6 files changed, 163 insertions(+), 24 deletions(-)
diff --git a/dev-support/pysubmarine/Dockerfile
b/dev-support/pysubmarine/Dockerfile
new file mode 100644
index 0000000..90878f4
--- /dev/null
+++ b/dev-support/pysubmarine/Dockerfile
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:18.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends apt-utils
+# Install JAVA
+RUN apt-get -q update \
+ && apt-get -q install -y --no-install-recommends openjdk-8-jdk
libbcprov-java \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+# Install Docker
+RUN apt-get update && \
+ apt-get -y install apt-transport-https ca-certificates curl
software-properties-common && \
+ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
+ add-apt-repository "deb [arch=amd64]
https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \
+ apt-get update && \
+ apt-get -y install docker-ce
+# So no need to mount host's /var/run/docker.sock, dockerd will create in
local fs
+VOLUME /var/lib/docker
+VOLUME /var/lib/docker.sock
+
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+ENV JRE_HOME /usr/lib/jvm/java-8-openjdk-amd64/jre
+
+# Install user tools
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ git make libgtest-dev cmake wget unzip libtinfo-dev libz-dev \
+ libcurl4-openssl-dev libopenblas-dev g++ sudo \
+ apt-transport-https curl vim ca-certificates maven
+
+WORKDIR /root
+
+# Intstall conda
+ADD install-conda.sh /usr/local
+ADD bootstrap.sh /usr/local
+RUN chmod 755 /usr/local/install-conda.sh && \
+ bash /usr/local/install-conda.sh
+
+WORKDIR /workspace
diff --git a/dev-support/pysubmarine/bootstrap.sh
b/dev-support/pysubmarine/bootstrap.sh
new file mode 100755
index 0000000..bd240b2
--- /dev/null
+++ b/dev-support/pysubmarine/bootstrap.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -euo pipefail
+
+COMMAND=("$@")
+
+getent group "${CI_BUILD_GID}" || addgroup --gid "${CI_BUILD_GID}"
"${CI_BUILD_GROUP}"
+getent passwd "${CI_BUILD_UID}" || adduser --gid "${CI_BUILD_GID}" --uid
"${CI_BUILD_UID}" \
+ --gecos "${CI_BUILD_USER} (generated by with_the_same_user script)" \
+ --disabled-password --home "${CI_BUILD_HOME}" --quiet "${CI_BUILD_USER}"
+usermod -a -G sudo "${CI_BUILD_USER}"
+echo "${CI_BUILD_USER} ALL=(ALL) NOPASSWD:ALL" >
/etc/sudoers.d/90-nopasswd-sudo
+# shellcheck disable=SC2024
+sudo nohup dockerd --host=unix:///var/run/docker.sock > /var/log/dockerd.log
2>&1 &
+sudo usermod -aG docker "${CI_BUILD_USER}"
+
+sudo -u "#${CI_BUILD_UID}" --preserve-env \
+HOME="${CI_BUILD_HOME}" \
+PATH="${PATH}" \
+PYTHONPATH="${PYTHONPATH}" \
+"${COMMAND[@]}"
diff --git a/submarine-sdk/pysubmarine/github-actions/lint.sh
b/dev-support/pysubmarine/install-conda.sh
old mode 100755
new mode 100644
similarity index 52%
copy from submarine-sdk/pysubmarine/github-actions/lint.sh
copy to dev-support/pysubmarine/install-conda.sh
index 07d92c0..be6a7cd
--- a/submarine-sdk/pysubmarine/github-actions/lint.sh
+++ b/dev-support/pysubmarine/install-conda.sh
@@ -14,29 +14,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -ex
+set -euo pipefail
+set -x
-FWDIR="$(cd "$(dirname "$0")"; pwd)"
-cd "$FWDIR"
-cd ..
-
-pycodestyle --max-line-length=100 -- submarine tests
-pylint --ignore experiment --msg-template="{path} ({line},{column}): [{msg_id}
{symbol}] {msg}" --rcfile=pylintrc -- submarine tests
-./github-actions/auto-format.sh
-
-GIT_STATUS="$(git status --porcelain)"
-GIT_DIFF="$(git diff)"
-if [ "$GIT_STATUS" ]; then
- echo "Code is not formatted by yapf and isort. Please run
./github-actions/auto-format.sh"
- echo "Git status is"
- echo
"------------------------------------------------------------------"
- echo "$GIT_STATUS"
- echo "Git diff is"
- echo
"------------------------------------------------------------------"
- echo "$GIT_DIFF"
- exit 1
+wget https://repo.anaconda.com/archive/Anaconda3-2020.02-Linux-x86_64.sh -O
"$HOME"/anaconda.sh;
+bash "$HOME"/anaconda.sh -b -p /usr/bin/anaconda
+export PATH="/usr/bin/anaconda/bin:$PATH"
+cd "$HOME"
+# Useful for debugging any issues with conda
+conda info -a
+if [[ -n "${PYTHON_VERSION:-}" ]]; then
+ conda create -q -n submarine-dev python="$PYTHON_VERSION"
else
- echo "Test successful"
+ conda create -q -n submarine-dev python=3.6
fi
-set +ex
+source activate submarine-dev
+python --version
+pip install --upgrade pip
+
+# Install pysubmarine
+git clone https://github.com/apache/submarine.git
+cd submarine/submarine-sdk/pysubmarine
+pip install .
+pip install -r github-actions/lint-requirements.txt
+pip install -r github-actions/test-requirements.txt
diff --git a/dev-support/pysubmarine/run-pysubmarine-ci.sh
b/dev-support/pysubmarine/run-pysubmarine-ci.sh
new file mode 100755
index 0000000..317e491
--- /dev/null
+++ b/dev-support/pysubmarine/run-pysubmarine-ci.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FWDIR="$(cd "$(dirname "$0")"; pwd)"
+cd "$FWDIR"
+
+# shellcheck disable=SC2034
+SUBMARINE_PROJECT_PATH="$FWDIR/../.."
+
+# build image
+echo "Start building the mini-submarine docker image..."
+docker build --tag pysubmarine-ci .
+
+docker run --rm --pid=host \
+ --privileged \
+ -v "$SUBMARINE_PROJECT_PATH":/workspace \
+ -e "CI_BUILD_HOME=/" \
+ -e "CI_BUILD_USER=$(id -u -n)" \
+ -e "CI_BUILD_UID=$(id -u)" \
+ -e "CI_BUILD_GROUP=$(id -g -n)" \
+ -e "CI_BUILD_GID=$(id -g)" \
+ -e "PATH=/usr/bin/anaconda/envs/submarine-dev/bin:${PATH}" \
+ -e
"PYTHONPATH=python:/usr/bin/anaconda/envs/submarine-dev/bin/python"\
+ -it pysubmarine-ci \
+ /bin/bash --login /usr/local/bootstrap.sh bash
diff --git a/docs/submarine-sdk/pysubmarine/development.md
b/docs/submarine-sdk/pysubmarine/development.md
index 29e5031..6043d45 100644
--- a/docs/submarine-sdk/pysubmarine/development.md
+++ b/docs/submarine-sdk/pysubmarine/development.md
@@ -28,6 +28,16 @@ pip install -r test-requirements.txt
# Installs pysubmarine from current checkout
pip install ./submarine-sdk/pysubmarine
```
+### PySubmarine Docker
+We also use docker to provide build environments for CI, development,
+generate python sdk from swagger.
+```bash
+./run-pysubmarine-ci.sh
+```
+The script does the following things:
+- Start an interactive bash session
+- Mount submarine directory to /workspace and set it as home
+- Switch user to be the same user that calls the `run-pysubmarine-ci.sh`
### Coding Style
- Use [yapf](https://github.com/google/yapf) to format Python code
- yapf style is configured in `.style.yapf` file
diff --git a/submarine-sdk/pysubmarine/github-actions/lint.sh
b/submarine-sdk/pysubmarine/github-actions/lint.sh
index 07d92c0..2bbcb13 100755
--- a/submarine-sdk/pysubmarine/github-actions/lint.sh
+++ b/submarine-sdk/pysubmarine/github-actions/lint.sh
@@ -25,13 +25,14 @@ pylint --ignore experiment --msg-template="{path}
({line},{column}): [{msg_id} {
./github-actions/auto-format.sh
GIT_STATUS="$(git status --porcelain)"
-GIT_DIFF="$(git diff)"
+# Only check the files in ./pysubmarine
+GIT_DIFF="$(git diff .)"
if [ "$GIT_STATUS" ]; then
echo "Code is not formatted by yapf and isort. Please run
./github-actions/auto-format.sh"
echo "Git status is"
echo
"------------------------------------------------------------------"
echo "$GIT_STATUS"
- echo "Git diff is"
+ echo "Git diff ./pysubmarine is"
echo
"------------------------------------------------------------------"
echo "$GIT_DIFF"
exit 1
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
