This is an automated email from the ASF dual-hosted git repository.
yjc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
The following commit(s) were added to refs/heads/master by this push:
new dd7f3d5 build: split hive and presto tests to separate jobs (#10821)
dd7f3d5 is described below
commit dd7f3d54023a21f9c42b3303b11d78fdb3ba2749
Author: Jesse Yang <[email protected]>
AuthorDate: Thu Sep 10 13:12:00 2020 -0700
build: split hive and presto tests to separate jobs (#10821)
* build: split hive and presto tests to separate jobs
* Must install doc dependencies, too
---
...prefer_typescript.yml => prefer-typescript.yml} | 0
.github/workflows/superset-docs.yml | 36 ++++
.github/workflows/superset-e2e.yml | 3 +-
.github/workflows/superset-frontend.yml | 3 +-
.github/workflows/superset-python.yml | 207 +--------------------
.github/workflows/superset-translations.yml | 28 ++-
.../{superset-docker.yml => test-docker.yml} | 2 +-
.github/workflows/test-hive.yml | 61 ++++++
.github/workflows/test-presto.yml | 64 +++++++
9 files changed, 191 insertions(+), 213 deletions(-)
diff --git a/.github/workflows/prefer_typescript.yml
b/.github/workflows/prefer-typescript.yml
similarity index 100%
rename from .github/workflows/prefer_typescript.yml
rename to .github/workflows/prefer-typescript.yml
diff --git a/.github/workflows/superset-docs.yml
b/.github/workflows/superset-docs.yml
new file mode 100644
index 0000000..6962837
--- /dev/null
+++ b/.github/workflows/superset-docs.yml
@@ -0,0 +1,36 @@
+# Python linting and docs
+name: Docs
+
+on: [push, pull_request]
+
+jobs:
+ lint:
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ python-version: [3.7]
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+ - name: Setup Python
+ uses: actions/[email protected]
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ uses: apache-superset/cached-dependencies@b90713b
+ with:
+ run: |
+ apt-get-install
+ pip-upgrade
+ pip install -r requirements/testing.txt
+ pip install -r requirements/documentation.txt
+
+ - name: pylint
+ # `-j 0` run Pylint in parallel
+ run: pylint -j 0 superset
+
+ - name: pre-commit
+ run: pre-commit run --all-files
+
+ - name: Build documentation
+ run: sphinx-build -b html docs _build/html -W
diff --git a/.github/workflows/superset-e2e.yml
b/.github/workflows/superset-e2e.yml
index e15c7c6..dfa6910 100644
--- a/.github/workflows/superset-e2e.yml
+++ b/.github/workflows/superset-e2e.yml
@@ -3,8 +3,7 @@ name: E2E
on: [push, pull_request]
jobs:
- cypress:
- name: Cypress
+ Cypress:
runs-on: ubuntu-18.04
strategy:
fail-fast: true
diff --git a/.github/workflows/superset-frontend.yml
b/.github/workflows/superset-frontend.yml
index 837af46..8d10e0b 100644
--- a/.github/workflows/superset-frontend.yml
+++ b/.github/workflows/superset-frontend.yml
@@ -3,8 +3,7 @@ name: Frontend
on: [push, pull_request]
jobs:
- frontend-build:
- name: build
+ build:
runs-on: ubuntu-18.04
steps:
- name: Checkout code
diff --git a/.github/workflows/superset-python.yml
b/.github/workflows/superset-python.yml
index f37463a..5f7e5e1 100644
--- a/.github/workflows/superset-python.yml
+++ b/.github/workflows/superset-python.yml
@@ -1,214 +1,9 @@
+# Python unit tests
name: Python
on: [push, pull_request]
jobs:
- lint:
- runs-on: ubuntu-18.04
- strategy:
- matrix:
- python-version: [3.7]
- steps:
- - name: Checkout code
- uses: actions/checkout@v2
- - name: Setup Python
- uses: actions/[email protected]
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install dependencies
- uses: apache-superset/cached-dependencies@b90713b
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install -r requirements/testing.txt
- - name: pylint
- # `-j 0` run Pylint in parallel
- run: pylint -j 0 superset
-
- pre-commit:
- runs-on: ubuntu-18.04
- strategy:
- matrix:
- python-version: [3.7]
- steps:
- - name: Checkout code
- uses: actions/checkout@v2
- - name: Setup Python
- uses: actions/[email protected]
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install dependencies
- uses: apache-superset/cached-dependencies@b90713b
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install -r requirements/integration.txt
- - name: pre-commit
- run: pre-commit run --all-files
-
- docs:
- runs-on: ubuntu-18.04
- strategy:
- matrix:
- python-version: [3.7]
- steps:
- - name: Checkout code
- uses: actions/checkout@v2
- - name: Setup Python
- uses: actions/[email protected]
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install dependencies
- uses: apache-superset/cached-dependencies@b90713b
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install -r requirements/documentation.txt
- - name: Build documentation
- run: sphinx-build -b html docs _build/html -W
-
- babel-extract:
- runs-on: ubuntu-18.04
- strategy:
- matrix:
- python-version: [3.7]
- steps:
- - name: Checkout code
- uses: actions/checkout@v2
- - name: Setup Python
- uses: actions/[email protected]
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install dependencies
- uses: apache-superset/cached-dependencies@b90713b
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install -r requirements/documentation.txt
- - name: Test babel extraction
- run: flask fab babel-extract --target superset/translations --output
superset/translations/messages.pot --config superset/translations/babel.cfg -k
_,__,t,tn,tct
-
- test-postgres-presto:
- runs-on: ubuntu-18.04
- strategy:
- matrix:
- # run unit tests in multiple version just for fun
- python-version: [3.8]
- env:
- PYTHONPATH: ${{ github.workspace }}
- SUPERSET_CONFIG: tests.superset_test_config
- REDIS_PORT: 16379
- SUPERSET__SQLALCHEMY_DATABASE_URI:
- postgresql+psycopg2://superset:[email protected]:15432/superset
- SUPERSET__SQLALCHEMY_EXAMPLES_URI:
- presto://localhost:15433/memory/default
- services:
- postgres:
- image: postgres:10-alpine
- env:
- POSTGRES_USER: superset
- POSTGRES_PASSWORD: superset
- ports:
- # Use custom ports for services to avoid accidentally connecting to
- # GitHub action runner's default installations
- - 15432:5432
- presto:
- image: prestosql/presto:339
- env:
- POSTGRES_USER: superset
- POSTGRES_PASSWORD: superset
- ports:
- # Use custom ports for services to avoid accidentally connecting to
- # GitHub action runner's default installations
- - 15433:8080
- redis:
- image: redis:5-alpine
- ports:
- - 16379:6379
- steps:
- - uses: actions/checkout@v2
- - name: Setup Python
- uses: actions/[email protected]
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install dependencies
- uses: apache-superset/cached-dependencies@b90713b
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install -r requirements/testing.txt
- setup-postgres
- - name: Run celery
- run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- - name: Python unit tests (PostgreSQL)
- run: |
- ./scripts/python_tests.sh
- - name: Upload code coverage
- run: |
- bash <(curl -s https://codecov.io/bash) -cF python
-
- test-postgres-hive:
- runs-on: ubuntu-18.04
- strategy:
- matrix:
- # run unit tests in multiple version just for fun
- python-version: [3.7, 3.8]
- env:
- PYTHONPATH: ${{ github.workspace }}
- SUPERSET_CONFIG: tests.superset_test_config
- REDIS_PORT: 16379
- SUPERSET__SQLALCHEMY_DATABASE_URI:
- postgresql+psycopg2://superset:[email protected]:15432/superset
- SUPERSET__SQLALCHEMY_EXAMPLES_URI: hive://localhost:10000/default
- UPLOAD_FOLDER: /tmp/.superset/uploads/
- services:
- postgres:
- image: postgres:10-alpine
- env:
- POSTGRES_USER: superset
- POSTGRES_PASSWORD: superset
- ports:
- # Use custom ports for services to avoid accidentally connecting to
- # GitHub action runner's default installations
- - 15432:5432
- redis:
- image: redis:5-alpine
- ports:
- - 16379:6379
- steps:
- - uses: actions/checkout@v2
- - name: Create csv upload directory
- run: sudo mkdir -p /tmp/.superset/uploads
- - name: Give write access to the csv upload directory
- run: sudo chown -R $USER:$USER /tmp/.superset
- - name: Start hadoop and hive
- run: docker-compose -f scripts/databases/hive/docker-compose.yml up -d
- - name: Setup Python
- uses: actions/[email protected]
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install dependencies
- uses: apache-superset/cached-dependencies@b90713b
- with:
- run: |
- apt-get-install
- pip-upgrade
- pip install -r requirements/testing.txt
- setup-postgres
- - name: Run celery
- run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- - name: Python unit tests (PostgreSQL)
- run: |
- ./scripts/python_tests.sh
- - name: Upload code coverage
- run: |
- bash <(curl -s https://codecov.io/bash) -cF python
-
test-postgres:
runs-on: ubuntu-18.04
strategy:
diff --git a/.github/workflows/superset-translations.yml
b/.github/workflows/superset-translations.yml
index 0fbcb1a..55281fa 100644
--- a/.github/workflows/superset-translations.yml
+++ b/.github/workflows/superset-translations.yml
@@ -3,8 +3,7 @@ name: Translations
on: [push, pull_request]
jobs:
- frontend-build:
- name: build
+ frontend-check:
runs-on: ubuntu-18.04
steps:
- name: Checkout code
@@ -17,3 +16,28 @@ jobs:
working-directory: ./superset-frontend
run: |
npm run check-translation
+
+ babel-extract:
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ python-version: [3.7]
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+ - name: Setup Python
+ uses: actions/[email protected]
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ uses: apache-superset/cached-dependencies@b90713b
+ with:
+ run: |
+ apt-get-install
+ pip-upgrade
+ pip install -r requirements/documentation.txt
+ - name: Test babel extraction
+ run: |
+ flask fab babel-extract --target superset/translations \
+ --output superset/translations/messages.pot \
+ --config superset/translations/babel.cfg -k _,__,t,tn,tct
diff --git a/.github/workflows/superset-docker.yml
b/.github/workflows/test-docker.yml
similarity index 97%
rename from .github/workflows/superset-docker.yml
rename to .github/workflows/test-docker.yml
index a9fae05..f2e0498 100644
--- a/.github/workflows/superset-docker.yml
+++ b/.github/workflows/test-docker.yml
@@ -1,4 +1,4 @@
-name: DOCKER
+name: Docker
on: [push, pull_request]
diff --git a/.github/workflows/test-hive.yml b/.github/workflows/test-hive.yml
new file mode 100644
index 0000000..2571f4a
--- /dev/null
+++ b/.github/workflows/test-hive.yml
@@ -0,0 +1,61 @@
+name: Hive
+
+on: [push, pull_request]
+
+jobs:
+ test-postgres-hive:
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ # run unit tests in multiple version just for fun
+ python-version: [3.7, 3.8]
+ env:
+ PYTHONPATH: ${{ github.workspace }}
+ SUPERSET_CONFIG: tests.superset_test_config
+ REDIS_PORT: 16379
+ SUPERSET__SQLALCHEMY_DATABASE_URI:
+ postgresql+psycopg2://superset:[email protected]:15432/superset
+ SUPERSET__SQLALCHEMY_EXAMPLES_URI: hive://localhost:10000/default
+ UPLOAD_FOLDER: /tmp/.superset/uploads/
+ services:
+ postgres:
+ image: postgres:10-alpine
+ env:
+ POSTGRES_USER: superset
+ POSTGRES_PASSWORD: superset
+ ports:
+ # Use custom ports for services to avoid accidentally connecting to
+ # GitHub action runner's default installations
+ - 15432:5432
+ redis:
+ image: redis:5-alpine
+ ports:
+ - 16379:6379
+ steps:
+ - uses: actions/checkout@v2
+ - name: Create csv upload directory
+ run: sudo mkdir -p /tmp/.superset/uploads
+ - name: Give write access to the csv upload directory
+ run: sudo chown -R $USER:$USER /tmp/.superset
+ - name: Start hadoop and hive
+ run: docker-compose -f scripts/databases/hive/docker-compose.yml up -d
+ - name: Setup Python
+ uses: actions/[email protected]
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ uses: apache-superset/cached-dependencies@b90713b
+ with:
+ run: |
+ apt-get-install
+ pip-upgrade
+ pip install -r requirements/testing.txt
+ setup-postgres
+ - name: Run celery
+ run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
+ - name: Python unit tests (PostgreSQL)
+ run: |
+ ./scripts/python_tests.sh
+ - name: Upload code coverage
+ run: |
+ bash <(curl -s https://codecov.io/bash) -cF python
diff --git a/.github/workflows/test-presto.yml
b/.github/workflows/test-presto.yml
new file mode 100644
index 0000000..14ac0a8
--- /dev/null
+++ b/.github/workflows/test-presto.yml
@@ -0,0 +1,64 @@
+name: Presto
+
+on: [push, pull_request]
+
+jobs:
+ test-postgres-presto:
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ # run unit tests in multiple version just for fun
+ python-version: [3.8]
+ env:
+ PYTHONPATH: ${{ github.workspace }}
+ SUPERSET_CONFIG: tests.superset_test_config
+ REDIS_PORT: 16379
+ SUPERSET__SQLALCHEMY_DATABASE_URI:
+ postgresql+psycopg2://superset:[email protected]:15432/superset
+ SUPERSET__SQLALCHEMY_EXAMPLES_URI:
+ presto://localhost:15433/memory/default
+ services:
+ postgres:
+ image: postgres:10-alpine
+ env:
+ POSTGRES_USER: superset
+ POSTGRES_PASSWORD: superset
+ ports:
+ # Use custom ports for services to avoid accidentally connecting to
+ # GitHub action runner's default installations
+ - 15432:5432
+ presto:
+ image: prestosql/presto:339
+ env:
+ POSTGRES_USER: superset
+ POSTGRES_PASSWORD: superset
+ ports:
+ # Use custom ports for services to avoid accidentally connecting to
+ # GitHub action runner's default installations
+ - 15433:8080
+ redis:
+ image: redis:5-alpine
+ ports:
+ - 16379:6379
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup Python
+ uses: actions/[email protected]
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ uses: apache-superset/cached-dependencies@b90713b
+ with:
+ run: |
+ apt-get-install
+ pip-upgrade
+ pip install -r requirements/testing.txt
+ setup-postgres
+ - name: Run celery
+ run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
+ - name: Python unit tests (PostgreSQL)
+ run: |
+ ./scripts/python_tests.sh
+ - name: Upload code coverage
+ run: |
+ bash <(curl -s https://codecov.io/bash) -cF python
