This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new e3a0839 Security scans are also selective now (#11674)
e3a0839 is described below
commit e3a0839e2139a32202a1096155d6203ce67ffaf8
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Oct 20 12:19:16 2020 +0200
Security scans are also selective now (#11674)
The security scans take a long time, especially for python code
- it is about ~18 minutes now. This PR reduces strain on the
GitHub actions by only running the scan in pull requests
when any of python/javascript code changed respectively.
---
.github/workflows/codeql-analysis.yml | 51 ++++++++++++++++++++++++++++++++---
CI.rst | 12 +++++++++
scripts/ci/selective_ci_checks.sh | 46 +++++++++++++++++++++++++++++++
3 files changed, 106 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/codeql-analysis.yml
b/.github/workflows/codeql-analysis.yml
index 570c118..292bfed 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -10,10 +10,36 @@ on:
- cron: '0 2 * * *'
jobs:
+ selective-checks:
+ name: Selective checks
+ runs-on: ubuntu-latest
+ if: github.repository == 'apache/airflow'
+ outputs:
+ needs-python-scans: ${{
steps.selective-checks.outputs.needs-python-scans }}
+ needs-javascript-scans: ${{
steps.selective-checks.outputs.needs-javascript-scans }}
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v2
+ with:
+ fetch-depth: 2
+ - name: Selective checks
+ id: selective-checks
+ env:
+ EVENT_NAME: ${{ github.event_name }}
+ INCOMING_COMMIT_SHA: ${{ github.sha }}
+ run: |
+ if [[ ${EVENT_NAME} == "pull_request" ]]; then
+ # Run selective checks
+ ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}"
+ else
+ # Run all checks
+ ./scripts/ci/selective_ci_checks.sh
+ fi
+
analyze:
name: Analyze
runs-on: ubuntu-latest
-
+ needs: [selective-checks]
strategy:
fail-fast: false
matrix:
@@ -21,7 +47,6 @@ jobs:
# Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript',
'python']
language: ['python', 'javascript']
- if: github.repository == 'apache/airflow'
steps:
- name: Checkout repository
uses: actions/checkout@v2
@@ -29,11 +54,19 @@ jobs:
# We must fetch at least the immediate parents so that if this is
# a pull request then we can checkout the head.
fetch-depth: 2
+ if: |
+ github.repository == 'apache/airflow' &&
+ (matrix.language == 'python' &&
needs.selective-checks.outputs.needs-python-scans == 'true' ||
+ matrix.language == 'javascript' &&
needs.selective-checks.outputs.needs-javascript-scans == 'true')
# If this run was triggered by a pull request event, then checkout
# the head of the pull request instead of the merge commit.
- run: git checkout HEAD^2
- if: github.event_name == 'pull_request'
+ if: |
+ github.event_name == 'pull_request' &&
+ github.repository == 'apache/airflow' &&
+ (matrix.language == 'python' &&
needs.selective-checks.outputs.needs-python-scans == 'true' ||
+ matrix.language == 'javascript' &&
needs.selective-checks.outputs.needs-javascript-scans == 'true')
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
@@ -44,11 +77,23 @@ jobs:
# By default, queries listed here will override any specified in a
config file.
# Prefix the list here with "+" to use these queries and those in
the config file.
# queries: ./path/to/local/query, your-org/your-repo/queries@main
+ if: |
+ github.repository == 'apache/airflow' &&
+ (matrix.language == 'python' &&
needs.selective-checks.outputs.needs-python-scans == 'true' ||
+ matrix.language == 'javascript' &&
needs.selective-checks.outputs.needs-javascript-scans == 'true')
# Autobuild attempts to build any compiled languages (C/C++, C#, or
Java).
# If this step fails, then you should remove it and run the build
manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v1
+ if: |
+ github.repository == 'apache/airflow' &&
+ (matrix.language == 'python' &&
needs.selective-checks.outputs.needs-python-scans == 'true' ||
+ matrix.language == 'javascript' &&
needs.selective-checks.outputs.needs-javascript-scans == 'true')
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1
+ if: |
+ github.repository == 'apache/airflow' &&
+ (matrix.language == 'python' &&
needs.selective-checks.outputs.needs-python-scans == 'true' ||
+ matrix.language == 'javascript' &&
needs.selective-checks.outputs.needs-javascript-scans == 'true')
diff --git a/CI.rst b/CI.rst
index b936c7d..14c0b45 100644
--- a/CI.rst
+++ b/CI.rst
@@ -691,6 +691,13 @@ delete old artifacts that are > 7 days old. It only runs
for the 'apache/airflow
We also have a script that can help to clean-up the old artifacts:
`remove_artifacts.sh <dev/remove_artifacts.sh>`_
+CodeQL scan
+-----------
+
+The CodeQL security scan uses GitHub security scan framework to scan our code
for security violations.
+It is run for javascript and python code.
+
+
Selective CI Checks
===================
@@ -775,6 +782,11 @@ The logic implemented for the changes works as follows:
commit - unlike pylint/flake8/mypy, those static checks are per-file based
and they should not miss any
important change.
+Similarly to selective tests we also run selective security scans. In Pull
requests,
+the Python scan will only run when there is a python code change and
javascript scan will only run if
+there is a javascript or yarn.lock file change. For master builds, all scans
are always executed.
+
+
Naming conventions for stored images
====================================
diff --git a/scripts/ci/selective_ci_checks.sh
b/scripts/ci/selective_ci_checks.sh
index 28efb3d..7b3b584 100755
--- a/scripts/ci/selective_ci_checks.sh
+++ b/scripts/ci/selective_ci_checks.sh
@@ -113,6 +113,15 @@ function needs_api_tests() {
initialization::ga_output needs-api-tests "${@}"
}
+function needs_javascript_scans() {
+ initialization::ga_output needs-javascript-scans "${@}"
+}
+
+function needs_python_scans() {
+ initialization::ga_output needs-python-scans "${@}"
+}
+
+
function set_test_types() {
initialization::ga_output test-types "${@}"
}
@@ -127,6 +136,8 @@ readonly ALL_TESTS
function set_outputs_run_everything_and_exit() {
needs_api_tests "true"
needs_helm_tests "true"
+ needs_javascript_scans "true"
+ needs_python_scans "true"
run_tests "true"
run_kubernetes_tests "true"
set_test_types "${ALL_TESTS}"
@@ -142,6 +153,10 @@ function set_outputs_run_all_tests() {
}
function set_output_skip_all_tests_and_exit() {
+ needs_api_tests "false"
+ needs_helm_tests "false"
+ needs_javascript_scans "false"
+ needs_python_scans "false"
run_tests "false"
run_kubernetes_tests "false"
set_test_types ""
@@ -185,6 +200,35 @@ function count_changed_files() {
echo "${CHANGED_FILES}" | grep -c -E "$(get_regexp_from_patterns)" || true
}
+function check_if_python_security_scans_should_be_run() {
+ local pattern_array=(
+ "^airflow/.*\.py"
+ "^setup.py"
+ )
+ show_changed_files
+
+ if [[ $(count_changed_files) == "0" ]]; then
+ needs_python_scans "false"
+ else
+ needs_python_scans "true"
+ fi
+}
+
+function check_if_javascript_security_scans_should_be_run() {
+ local pattern_array=(
+ "^airflow/.*\.js"
+ "^airflow/.*\.lock"
+ )
+ show_changed_files
+
+ if [[ $(count_changed_files) == "0" ]]; then
+ needs_javascript_scans "false"
+ else
+ needs_javascript_scans "true"
+ fi
+}
+
+
function check_if_api_tests_should_be_run() {
local pattern_array=(
"^airflow/api"
@@ -416,6 +460,8 @@ check_if_docs_should_be_generated
check_if_helm_tests_should_be_run
check_if_api_tests_should_be_run
check_if_tests_are_needed_at_all
+check_if_javascript_security_scans_should_be_run
+check_if_python_security_scans_should_be_run
get_count_all_files
get_count_api_files
get_count_cli_files
