aaltay commented on a change in pull request #12318:
URL: https://github.com/apache/beam/pull/12318#discussion_r458272138
##########
File path: .github/workflows/build_wheels.yml
##########
@@ -193,29 +196,232 @@ jobs:
GITHUB_BASE_REF=$GITHUB_BASE_REF
EOF
echo $(cat github_action_info)
- gsutil cp -a public-read github_action_info ${{ env.GCP_PATH }}
+ gsutil cp -a public-read github_action_info ${{
env.GCP_WHEELS_STAGING_PATH }}
- name: Upload GitHub event file to GCS bucket
- run: gsutil cp -a public-read ${GITHUB_EVENT_PATH} ${{ env.GCP_PATH }}
+ run: gsutil cp -a public-read ${GITHUB_EVENT_PATH} ${{
env.GCP_WHEELS_STAGING_PATH }}
list_files_on_gcs:
name: List files on Google Cloud Storage Bucket
- needs: upload_wheels_to_gcs
+ needs: upload_python_sdk_wheels_to_gcs
runs-on: ubuntu-latest
- if: github.repository_owner == 'apache' && github.event_name !=
'pull_request'
+ if: github.repository_owner == 'apache' && (github.event_name == 'push' ||
github.event_name == 'schedule')
steps:
- name: Authenticate on GCP
uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
with:
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
- name: List file on Google Cloud Storage Bucket
- run: gsutil ls "${{ env.GCP_PATH }}*"
+ run: gsutil ls "${{ env.GCP_WHEELS_STAGING_PATH }}*"
+
+ python_unit_tests:
+ name: 'Python Unit Tests'
+ runs-on: ${{ matrix.os }}
+ if: github.event_name == 'push' || github.event_name == 'schedule'
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, windows-latest]
+ params: [
+ {"py_ver": "3.5", "tox_env": "py35"},
+ {"py_ver": "3.6", "tox_env": "py36"},
+ {"py_ver": "3.7", "tox_env": "py37"},
+ ]
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+ - name: Install python
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.params.py_ver }}
+ - name: Get build dependencies
+ working-directory: ./sdks/python
+ run: python -m pip install -r build-requirements.txt
+ - name: Install tox
+ run: pip install tox
+ - name: Run tests basic unix
+ if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')
+ working-directory: ./sdks/python
+ run: tox -c tox.ini -e ${{ matrix.params.tox_env }}
+ - name: Run tests basic windows
+ if: startsWith(matrix.os, 'windows') && (matrix.params.py_ver != '2.7')
+ working-directory: ./sdks/python
+ run: tox -c tox.ini -e ${{ matrix.params.tox_env }}-win
+
+ python_wordcount_direct_runner:
+ name: 'Python Wordcount Direct Runner'
+ runs-on: ${{ matrix.os }}
+ if: github.event_name == 'push' || github.event_name == 'schedule'
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, macos-latest, windows-latest]
+ python: [3.5, 3.6, 3.7]
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+ - name: Install python
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python }}
+ - name: Get build dependencies
+ working-directory: ./sdks/python
+ run: python -m pip install -r build-requirements.txt
+ - name: Install requirements
+ working-directory: ./sdks/python
+ run: pip install -e .
+ - name: Run WordCount
+ working-directory: ./sdks/python
+ run: python -m apache_beam.examples.wordcount --input MANIFEST.in
--output counts
+
+ python_wordcount_dataflow:
+ name: 'Python Wordcount Dataflow'
+ needs: build_python_sdk_source
+ runs-on: ${{ matrix.os }}
+ if: github.repository_owner == 'apache' && (github.event_name == 'push' ||
github.event_name == 'schedule')
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, macos-latest, windows-latest]
+ python: [3.5, 3.6, 3.7]
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+ - name: Install python
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python }}
+ - name: Download source from artifacts
+ uses: actions/download-artifact@v2
+ with:
+ name: source_gztar_zip
+ path: apache-beam-source
+ - name: Authenticate on GCP
+ uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+ with:
+ service_account_email: ${{ secrets.GCP_SA_EMAIL }}
+ service_account_key: ${{ secrets.GCP_SA_KEY }}
+ export_default_credentials: true
+ - name: Get build dependencies
+ working-directory: ./sdks/python
+ run: python -m pip install -r build-requirements.txt
+ - name: Install requirements
+ working-directory: ./sdks/python
+ run: pip install -e ".[gcp]"
+ - name: Run WordCount
+ working-directory: ./sdks/python
+ shell: bash
+ run: |
+ python -m apache_beam.examples.wordcount \
+ --input gs://dataflow-samples/shakespeare/kinglear.txt \
+ --output gs://${{ env.GCP_BUCKET }}/counts \
+ --runner DataflowRunner \
+ --project ${{ env.GCP_PROJECT_ID }} \
+ --region ${{ env.GCP_REGION }} \
+ --temp_location gs://${{ env.GCP_BUCKET }}/tmp/ \
+ --sdk_location "../../apache-beam-source/$(ls
../../apache-beam-source | grep '^apache-beam.*zip$')"
+
+ java_unit_tests:
+ name: 'Java Unit Tests'
+ runs-on: ${{ matrix.os }}
+ if: github.event_name == 'push' || github.event_name == 'schedule'
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, macos-latest, windows-latest]
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+ - name: Run tests
+ run: |
+ ./gradlew :sdks:java:core:test
+ ./gradlew :sdks:java:harness:test
+ ./gradlew :runners:core-java:test
+
+ java_wordcount_direct_runner:
+ name: 'Java Wordcount Direct Runner'
+ runs-on: ${{ matrix.os }}
+ if: github.event_name == 'push' || github.event_name == 'schedule'
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, macos-latest, windows-latest]
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+ - name: Get the WordCount Code
+ shell: bash
+ run: |
+ mvn archetype:generate \
+ -DarchetypeGroupId=org.apache.beam \
+ -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \
+ -DarchetypeVersion=2.22.0 \
+ -DgroupId=org.example \
+ -DartifactId=word-count-beam \
+ -Dversion="0.1" \
+ -Dpackage=org.apache.beam.examples \
+ -DinteractiveMode=false
+ - name: Run WordCount
+ working-directory: ./word-count-beam
+ shell: bash
+ run: |
+ mvn compile exec:java \
+ -Dexec.mainClass=org.apache.beam.examples.WordCount \
+ -Dexec.args="--inputFile=pom.xml --output=counts" \
+ -Pdirect-runner
+
+ java_wordcount_dataflow:
+ name: 'Java Wordcount Dataflow'
+ runs-on: ${{ matrix.os }}
+ if: github.repository_owner == 'apache' && (github.event_name == 'push' ||
github.event_name == 'schedule')
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, macos-latest, windows-latest]
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+ - name: Authenticate on GCP
+ uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+ with:
+ service_account_email: ${{ secrets.GCP_SA_EMAIL }}
+ service_account_key: ${{ secrets.GCP_SA_KEY }}
+ export_default_credentials: true
+ - name: Get the WordCount Code
+ shell: bash
+ run: |
+ mvn archetype:generate \
+ -DarchetypeGroupId=org.apache.beam \
+ -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \
+ -DarchetypeVersion=2.22.0 \
Review comment:
Should this version match what is at head?
##########
File path: .github/workflows/build_wheels.yml
##########
@@ -193,29 +196,232 @@ jobs:
GITHUB_BASE_REF=$GITHUB_BASE_REF
EOF
echo $(cat github_action_info)
- gsutil cp -a public-read github_action_info ${{ env.GCP_PATH }}
+ gsutil cp -a public-read github_action_info ${{
env.GCP_WHEELS_STAGING_PATH }}
- name: Upload GitHub event file to GCS bucket
- run: gsutil cp -a public-read ${GITHUB_EVENT_PATH} ${{ env.GCP_PATH }}
+ run: gsutil cp -a public-read ${GITHUB_EVENT_PATH} ${{
env.GCP_WHEELS_STAGING_PATH }}
list_files_on_gcs:
name: List files on Google Cloud Storage Bucket
- needs: upload_wheels_to_gcs
+ needs: upload_python_sdk_wheels_to_gcs
runs-on: ubuntu-latest
- if: github.repository_owner == 'apache' && github.event_name !=
'pull_request'
+ if: github.repository_owner == 'apache' && (github.event_name == 'push' ||
github.event_name == 'schedule')
steps:
- name: Authenticate on GCP
uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
with:
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
- name: List file on Google Cloud Storage Bucket
- run: gsutil ls "${{ env.GCP_PATH }}*"
+ run: gsutil ls "${{ env.GCP_WHEELS_STAGING_PATH }}*"
+
+ python_unit_tests:
+ name: 'Python Unit Tests'
+ runs-on: ${{ matrix.os }}
+ if: github.event_name == 'push' || github.event_name == 'schedule'
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, windows-latest]
+ params: [
+ {"py_ver": "3.5", "tox_env": "py35"},
+ {"py_ver": "3.6", "tox_env": "py36"},
+ {"py_ver": "3.7", "tox_env": "py37"},
+ ]
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v2
+ - name: Install python
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.params.py_ver }}
+ - name: Get build dependencies
+ working-directory: ./sdks/python
+ run: python -m pip install -r build-requirements.txt
+ - name: Install tox
+ run: pip install tox
+ - name: Run tests basic unix
+ if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')
+ working-directory: ./sdks/python
+ run: tox -c tox.ini -e ${{ matrix.params.tox_env }}
+ - name: Run tests basic windows
+ if: startsWith(matrix.os, 'windows') && (matrix.params.py_ver != '2.7')
Review comment:
py_ver 2.7 is not in the list of params. Do we need that check?
##########
File path: .github/workflows/build_wheels.yml
##########
@@ -193,29 +196,232 @@ jobs:
GITHUB_BASE_REF=$GITHUB_BASE_REF
EOF
echo $(cat github_action_info)
- gsutil cp -a public-read github_action_info ${{ env.GCP_PATH }}
+ gsutil cp -a public-read github_action_info ${{
env.GCP_WHEELS_STAGING_PATH }}
- name: Upload GitHub event file to GCS bucket
- run: gsutil cp -a public-read ${GITHUB_EVENT_PATH} ${{ env.GCP_PATH }}
+ run: gsutil cp -a public-read ${GITHUB_EVENT_PATH} ${{
env.GCP_WHEELS_STAGING_PATH }}
list_files_on_gcs:
name: List files on Google Cloud Storage Bucket
- needs: upload_wheels_to_gcs
+ needs: upload_python_sdk_wheels_to_gcs
runs-on: ubuntu-latest
- if: github.repository_owner == 'apache' && github.event_name !=
'pull_request'
+ if: github.repository_owner == 'apache' && (github.event_name == 'push' ||
github.event_name == 'schedule')
steps:
- name: Authenticate on GCP
uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
with:
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
- name: List file on Google Cloud Storage Bucket
- run: gsutil ls "${{ env.GCP_PATH }}*"
+ run: gsutil ls "${{ env.GCP_WHEELS_STAGING_PATH }}*"
+
+ python_unit_tests:
+ name: 'Python Unit Tests'
+ runs-on: ${{ matrix.os }}
+ if: github.event_name == 'push' || github.event_name == 'schedule'
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, windows-latest]
+ params: [
+ {"py_ver": "3.5", "tox_env": "py35"},
Review comment:
Is it possible to have the [3.5, 3.6, 3.7] list only in one place?
##########
File path: .github/workflows/build_wheels.yml
##########
@@ -193,29 +196,232 @@ jobs:
GITHUB_BASE_REF=$GITHUB_BASE_REF
EOF
echo $(cat github_action_info)
- gsutil cp -a public-read github_action_info ${{ env.GCP_PATH }}
+ gsutil cp -a public-read github_action_info ${{
env.GCP_WHEELS_STAGING_PATH }}
- name: Upload GitHub event file to GCS bucket
- run: gsutil cp -a public-read ${GITHUB_EVENT_PATH} ${{ env.GCP_PATH }}
+ run: gsutil cp -a public-read ${GITHUB_EVENT_PATH} ${{
env.GCP_WHEELS_STAGING_PATH }}
list_files_on_gcs:
name: List files on Google Cloud Storage Bucket
- needs: upload_wheels_to_gcs
+ needs: upload_python_sdk_wheels_to_gcs
runs-on: ubuntu-latest
- if: github.repository_owner == 'apache' && github.event_name !=
'pull_request'
+ if: github.repository_owner == 'apache' && (github.event_name == 'push' ||
github.event_name == 'schedule')
steps:
- name: Authenticate on GCP
uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
with:
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
- name: List file on Google Cloud Storage Bucket
- run: gsutil ls "${{ env.GCP_PATH }}*"
+ run: gsutil ls "${{ env.GCP_WHEELS_STAGING_PATH }}*"
+
+ python_unit_tests:
+ name: 'Python Unit Tests'
+ runs-on: ${{ matrix.os }}
+ if: github.event_name == 'push' || github.event_name == 'schedule'
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, windows-latest]
+ params: [
+ {"py_ver": "3.5", "tox_env": "py35"},
+ {"py_ver": "3.6", "tox_env": "py36"},
+ {"py_ver": "3.7", "tox_env": "py37"},
Review comment:
@tvalentyn - Should we add 3.8?
##########
File path: .github/workflows/build_wheels.yml
##########
@@ -193,29 +196,232 @@ jobs:
GITHUB_BASE_REF=$GITHUB_BASE_REF
EOF
echo $(cat github_action_info)
- gsutil cp -a public-read github_action_info ${{ env.GCP_PATH }}
+ gsutil cp -a public-read github_action_info ${{
env.GCP_WHEELS_STAGING_PATH }}
- name: Upload GitHub event file to GCS bucket
- run: gsutil cp -a public-read ${GITHUB_EVENT_PATH} ${{ env.GCP_PATH }}
+ run: gsutil cp -a public-read ${GITHUB_EVENT_PATH} ${{
env.GCP_WHEELS_STAGING_PATH }}
list_files_on_gcs:
name: List files on Google Cloud Storage Bucket
- needs: upload_wheels_to_gcs
+ needs: upload_python_sdk_wheels_to_gcs
runs-on: ubuntu-latest
- if: github.repository_owner == 'apache' && github.event_name !=
'pull_request'
+ if: github.repository_owner == 'apache' && (github.event_name == 'push' ||
github.event_name == 'schedule')
steps:
- name: Authenticate on GCP
uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
with:
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
- name: List file on Google Cloud Storage Bucket
- run: gsutil ls "${{ env.GCP_PATH }}*"
+ run: gsutil ls "${{ env.GCP_WHEELS_STAGING_PATH }}*"
+
+ python_unit_tests:
+ name: 'Python Unit Tests'
+ runs-on: ${{ matrix.os }}
+ if: github.event_name == 'push' || github.event_name == 'schedule'
Review comment:
Question: Do we want to run on this on every push? This might have quite
a bit of overlap with precommits and might be slow.
/cc @tvalentyn
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
