This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 4b3b648 SUBMARINE-416. [SDK] GitHub actions support pysubmarine
4b3b648 is described below
commit 4b3b6485057b8d25390f0a524e53fce82bab1018
Author: pingsutw <[email protected]>
AuthorDate: Mon Mar 9 23:36:46 2020 +0800
SUBMARINE-416. [SDK] GitHub actions support pysubmarine
### What is this PR for?
Support submarine SDK to use GitHub actions for a fast testing
### What type of PR is it?
[Improvement]
### Todos
* [ ] - Task
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-416
### How should this be tested?
https://travis-ci.org/pingsutw/hadoop-submarine/builds/660442033
https://github.com/pingsutw/hadoop-submarine/actions/runs/52676891
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: pingsutw <[email protected]>
Closes #217 from pingsutw/SUBMARINE-416 and squashes the following commits:
1d584da [pingsutw] SUBMARINE-416. [SDK] GitHub actions support pysubmarine
---
.github/workflows/python.yml | 49 ++++++++++++++++++++
.travis.yml | 14 +-----
.../lint-requirements.txt | 6 ++-
.../pysubmarine/{travis => github-actions}/lint.sh | 12 +++--
.../test-requirements.txt | 6 +--
submarine-sdk/pysubmarine/pylintrc | 1 +
.../pysubmarine/tests/ml/model/conftest.py | 4 +-
submarine-sdk/pysubmarine/travis/conda.sh | 54 ----------------------
8 files changed, 67 insertions(+), 79 deletions(-)
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
new file mode 100644
index 0000000..6a5a08d
--- /dev/null
+++ b/.github/workflows/python.yml
@@ -0,0 +1,49 @@
+name: python-sdk
+
+# Trigger the workflow on push or pull request
+on: [push, pull_request]
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ timeout-minutes: 120
+ strategy:
+ matrix:
+ python-version: [3.5,3.6,3.7]
+ tf-version: [1.14.0,1.15.0]
+ fail-fast: false
+ env:
+ PYTHON: ${{ matrix.python }}
+ steps:
+ - uses: actions/checkout@v1
+ - name: Verify MySQL connection from host
+ run: |
+ sudo apt-get install -y mysql-client
+ sudo service mysql restart
+ # default mysql account and password in github actions
+ mysql -uroot -proot -e "SHOW DATABASES"
+ - name: Setup submarine database
+ run: |
+ mysql -uroot -proot -e "create database submarine_test;"
+ mysql -uroot -proot -e "CREATE USER 'submarine_test'@'%' IDENTIFIED
BY 'password_test';"
+ mysql -uroot -proot -e "GRANT ALL PRIVILEGES ON *.* TO
'submarine_test'@'%';"
+ mysql -uroot -proot -e "use submarine_test; source
./docs/database/submarine.sql; show tables;"
+ - name: Setup python environment
+ uses: actions/setup-python@v1
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ run: |
+ pip install ./submarine-sdk/pysubmarine/.
+ pip install -q tensorflow==${{ matrix.tf-version }}
+ pip install -r
./submarine-sdk/pysubmarine/github-actions/test-requirements.txt
+ pip install -r
./submarine-sdk/pysubmarine/github-actions/lint-requirements.txt
+ - name: Check python sdk code style
+ if: matrix.python-version == '3.6'
+ run: |
+ ./submarine-sdk/pysubmarine/github-actions/lint.sh
+ - name: Test with pytest
+ run: |
+ pytest --cov=submarine -vs
+
+
diff --git a/.travis.yml b/.travis.yml
index c60df85..4178401 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -204,19 +204,7 @@ matrix:
# jdk: "openjdk8"
# dist: xenial
# env: PYTHON="3" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package
install -DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl
${EXCLUDE_SUBMITTER_K8S},${EXCLUDE_SUBMARINE},${EXCLUDE_WORKBENCH}"
TEST_MODULES="-pl $(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')"
TEST_PROJECTS=""
-
- - name: Test submarine sdk
- language: python
- python: 3.6
- dist: xenial
- env: NAME="Test Python SDK" TRAVIS_PYTHON_VERSION="3.6"
- install:
- - source ./submarine-sdk/pysubmarine/travis/conda.sh
- - pip install -r
./submarine-sdk/pysubmarine/travis/lint-requirements.txt
- script:
- - ./submarine-sdk/pysubmarine/travis/lint.sh
- - pytest --cov=submarine -vs
-
+
- name: Test submarine workbench-web VUE
language: node_js
node_js:
diff --git a/submarine-sdk/pysubmarine/travis/lint-requirements.txt
b/submarine-sdk/pysubmarine/github-actions/lint-requirements.txt
similarity index 79%
rename from submarine-sdk/pysubmarine/travis/lint-requirements.txt
rename to submarine-sdk/pysubmarine/github-actions/lint-requirements.txt
index e2f58bf..ab7efab 100644
--- a/submarine-sdk/pysubmarine/travis/lint-requirements.txt
+++ b/submarine-sdk/pysubmarine/github-actions/lint-requirements.txt
@@ -1,3 +1,5 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
@@ -11,6 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-prospector[with_pyroma]==0.12.7
pep8==1.7.1
-pylint==1.8.2
\ No newline at end of file
+pylint==2.4.4
+pycodestyle==2.5.0
\ No newline at end of file
diff --git a/submarine-sdk/pysubmarine/travis/lint.sh
b/submarine-sdk/pysubmarine/github-actions/lint.sh
similarity index 78%
rename from submarine-sdk/pysubmarine/travis/lint.sh
rename to submarine-sdk/pysubmarine/github-actions/lint.sh
index 220bfa8..d86e54c 100755
--- a/submarine-sdk/pysubmarine/travis/lint.sh
+++ b/submarine-sdk/pysubmarine/github-actions/lint.sh
@@ -1,11 +1,11 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
+# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
+# the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,11 +14,13 @@
# limitations under the License.
#!/usr/bin/env bash
-set -e
+set -ex
-FWDIR="$(cd "`dirname $0`"; pwd)"
+FWDIR="$(cd "$(dirname "$0")"; pwd)"
cd "$FWDIR"
cd ..
pycodestyle --max-line-length=100 -- submarine tests
pylint --msg-template="{path} ({line},{column}): [{msg_id} {symbol}] {msg}"
--rcfile=pylintrc -- submarine tests
+
+set +ex
diff --git a/submarine-sdk/pysubmarine/travis/test-requirements.txt
b/submarine-sdk/pysubmarine/github-actions/test-requirements.txt
similarity index 80%
rename from submarine-sdk/pysubmarine/travis/test-requirements.txt
rename to submarine-sdk/pysubmarine/github-actions/test-requirements.txt
index a24ba15..664f654 100644
--- a/submarine-sdk/pysubmarine/travis/test-requirements.txt
+++ b/submarine-sdk/pysubmarine/github-actions/test-requirements.txt
@@ -1,11 +1,11 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
+# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
+# the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/submarine-sdk/pysubmarine/pylintrc
b/submarine-sdk/pysubmarine/pylintrc
index b657436..6afd467 100644
--- a/submarine-sdk/pysubmarine/pylintrc
+++ b/submarine-sdk/pysubmarine/pylintrc
@@ -77,6 +77,7 @@ confidence=
# --disable=W".
disable=missing-docstring,
print-statement,
+ unnecessary-pass,
parameter-unpacking,
unpacking-in-except,
old-raise-syntax,
diff --git a/submarine-sdk/pysubmarine/tests/ml/model/conftest.py
b/submarine-sdk/pysubmarine/tests/ml/model/conftest.py
index 4623088..4db552e 100644
--- a/submarine-sdk/pysubmarine/tests/ml/model/conftest.py
+++ b/submarine-sdk/pysubmarine/tests/ml/model/conftest.py
@@ -29,8 +29,8 @@ LIBSVM_DATA = """1 1:0 2:0.051495 3:0.5 4:0.1 5:0.113437
6:0.874 7:0.01 8:0.08 9
@pytest.fixture
def get_model_param(tmpdir):
- data_file = os.path.join(tmpdir, "libsvm.txt")
- save_model_dir = os.path.join(tmpdir, "experiment")
+ data_file = os.path.join(str(tmpdir), "libsvm.txt")
+ save_model_dir = os.path.join(str(tmpdir), "experiment")
with open(data_file, "wt") as writer:
writer.write(LIBSVM_DATA)
diff --git a/submarine-sdk/pysubmarine/travis/conda.sh
b/submarine-sdk/pysubmarine/travis/conda.sh
deleted file mode 100644
index 1f8fc36..0000000
--- a/submarine-sdk/pysubmarine/travis/conda.sh
+++ /dev/null
@@ -1,54 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/usr/bin/env bash
-
-set -ex
-sudo mkdir -p /travis-install
-sudo chown travis /travis-install
-
-# We do this conditionally because it saves us some downloading if the
-# version is the same.
-if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
- wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh
-O /travis-install/miniconda.sh;
-else
- wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
-O /travis-install/miniconda.sh;
-fi
-
-bash /travis-install/miniconda.sh -b -p $HOME/miniconda
-export PATH="$HOME/miniconda/bin:$PATH"
-hash -r
-conda config --set always_yes yes --set changeps1 no
-# Useful for debugging any issues with conda
-conda info -a
-if [[ -n "$TRAVIS_PYTHON_VERSION" ]]; then
- conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION
-else
- conda create -q -n test-environment python=3.6
-fi
-source activate test-environment
-python --version
-pip install --upgrade pip
-pip install -r ./submarine-sdk/pysubmarine/travis/test-requirements.txt
-
-pip install ./submarine-sdk/pysubmarine/.
-export SUBMARINE_HOME=$(pwd)
-
-# Print current environment info
-pip list
-echo $SUBMARINE_HOME
-
-# Turn off trace output & exit-on-errors
-set +ex
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
