This is an automated email from the ASF dual-hosted git repository. cwylie pushed a commit to branch chill-travis in repository https://gitbox.apache.org/repos/asf/druid.git
commit 28e801200e1a1bfe3ba2cd3b79690c11a006bbb8 Author: Clint Wylie <[email protected]> AuthorDate: Fri May 21 01:27:51 2021 -0700 only skip stuff for PR builds, add some tests --- .idea/inspectionProfiles/Druid.xml | 6 +-- .travis.yml | 18 ++++---- check-test-suite.py => check_test_suite.py | 50 +++++++++++++--------- check_test_suite_test.py | 67 ++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 30 deletions(-) diff --git a/.idea/inspectionProfiles/Druid.xml b/.idea/inspectionProfiles/Druid.xml index 2da1cb7..bc1782f 100644 --- a/.idea/inspectionProfiles/Druid.xml +++ b/.idea/inspectionProfiles/Druid.xml @@ -59,6 +59,7 @@ <inspection_tool class="EqualsUsesNonFinalVariable" enabled="true" level="WARNING" enabled_by_default="true" /> <inspection_tool class="EqualsWhichDoesntCheckParameterClass" enabled="true" level="ERROR" enabled_by_default="true" /> <inspection_tool class="EqualsWithItself" enabled="true" level="ERROR" enabled_by_default="true" /> + <inspection_tool class="Eslint" enabled="true" level="WARNING" enabled_by_default="true" /> <inspection_tool class="FieldAccessNotGuarded" enabled="true" level="ERROR" enabled_by_default="true" /> <inspection_tool class="FieldCanBeLocal" enabled="true" level="WARNING" enabled_by_default="true"> <option name="EXCLUDE_ANNOS"> @@ -70,7 +71,6 @@ </option> <option name="IGNORE_FIELDS_USED_IN_MULTIPLE_METHODS" value="true" /> </inspection_tool> - <inspection_tool class="FieldMayBeFinal" enabled="true" level="WARNING" enabled_by_default="true" /> <inspection_tool class="FinalStaticMethod" enabled="true" level="ERROR" enabled_by_default="true" /> <inspection_tool class="FlowJSError" enabled="false" level="Non-TeamCity Error" enabled_by_default="false" /> <inspection_tool class="ForCanBeForeach" enabled="true" level="WARNING" enabled_by_default="true"> @@ -470,8 +470,8 @@ <inspection_tool class="XmlHighlighting" enabled="true" level="Non-TeamCity Warning" enabled_by_default="true" /> <inspection_tool class="XmlInvalidId" enabled="true" level="Non-TeamCity Error" enabled_by_default="true" /> <inspection_tool class="XmlPathReference" enabled="true" level="Non-TeamCity Error" enabled_by_default="true" /> - <inspection_tool class="unused" enabled="true" level="WARNING" enabled_by_default="true" isSelected="false"> - <scope name="UnusedInspectionsScope" level="ERROR" enabled="true" isSelected="false"> + <inspection_tool class="unused" enabled="true" level="WARNING" enabled_by_default="true"> + <scope name="UnusedInspectionsScope" level="ERROR" enabled="true"> <option name="LOCAL_VARIABLE" value="true" /> <option name="FIELD" value="true" /> <option name="METHOD" value="true" /> diff --git a/.travis.yml b/.travis.yml index 2792aaf..5178095 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ addons: # Add various options to make 'mvn install' fast and skip javascript compile (-Ddruid.console.skip=true) since it is not # needed. Depending on network speeds, "mvn -q install" may take longer than the default 10 minute timeout to print any # output. To compensate, use travis_wait to extend the timeout. -install: ./check-test-suite.py && travis_terminate 0 || echo 'Running maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} +install: ./check_test_suite.py && travis_terminate 0 || echo 'Running maven install...' && MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff -pl '!distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C && ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} # There are 3 stages of tests # 1. Tests - phase 1 @@ -88,7 +88,7 @@ jobs: install: skip before_script: &setup_generate_license - sudo apt-get update && sudo apt-get install python3 python3-pip python3-setuptools -y - - ./check-test-suite.py && travis_terminate 0 || echo 'Continuing setup' + - ./check_test_suite.py && travis_terminate 0 || echo 'Continuing setup' - pip3 install wheel # install wheel first explicitly - pip3 install pyyaml script: @@ -102,11 +102,15 @@ jobs: - distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2 - distribution/bin/check-licenses.py licenses.yaml target/license-reports + - name: "script checks" + install: skip + script: ./check_test_suite_test.py + - name: "(openjdk8) strict compilation" install: skip # Strict compilation requires more than 2 GB script: > - ./check-test-suite.py && travis_terminate 0 || MAVEN_OPTS='-Xmx3000m' ${MVN} clean -Pstrict compile test-compile --fail-at-end + ./check_test_suite.py && travis_terminate 0 || MAVEN_OPTS='-Xmx3000m' ${MVN} clean -Pstrict compile test-compile --fail-at-end -pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} - name: "analyze dependencies" @@ -133,7 +137,7 @@ jobs: - name: "intellij inspections" script: > - ./check-test-suite.py && travis_terminate 0 || docker run --rm + ./check_test_suite.py && travis_terminate 0 || docker run --rm -v $(pwd):/project -v ~/.m2:/home/inspect/.m2 ccaominh/intellij-inspect:1.0.0 @@ -301,7 +305,7 @@ jobs: install: skip stage: Tests - phase 1 script: - - ./check-test-suite.py && travis_terminate 0 || ${MVN} test -pl 'web-console' + - ./check_test_suite.py && travis_terminate 0 || ${MVN} test -pl 'web-console' after_success: - (cd web-console && travis_retry npm run codecov) # retry in case of network error @@ -309,14 +313,14 @@ jobs: before_install: *setup_generate_license install: web-console/script/druid build before_script: - - ./check-test-suite.py && travis_terminate 0 || echo 'Starting nvm install...' + - ./check_test_suite.py && travis_terminate 0 || echo 'Starting nvm install...' - nvm install 10.24.0 - web-console/script/druid start script: (cd web-console && npm run test-e2e) after_script: web-console/script/druid stop - name: "docs" - install: ./check-test-suite.py && travis_terminate 0 || (cd website && npm install) + install: ./check_test_suite.py && travis_terminate 0 || (cd website && npm install) script: |- (cd website && npm run lint && npm run spellcheck) || { echo " diff --git a/check-test-suite.py b/check_test_suite.py similarity index 71% rename from check-test-suite.py rename to check_test_suite.py index 5006e92..b067c27 100755 --- a/check-test-suite.py +++ b/check_test_suite.py @@ -27,7 +27,7 @@ always_run_jobs = ['license checks', '(openjdk8) packaging check', '(openjdk11) # of CI can be skipped ignore_prefixes = ['.github', '.idea', '.asf.yaml', '.backportrc.json', '.codecov.yml', '.dockerignore', '.gitignore', '.lgtm.yml', 'CONTRIBUTING.md', 'setup-hooks.sh', 'upload.sh', 'dev', 'distribution/docker', - 'distribution/asf-release-process-guide.md', '.travis.yml', 'check-test-suite.py'] + 'distribution/asf-release-process-guide.md', '.travis.yml', 'check_test_suite.py'] # these files are docs changes # if changes are limited to this set then we can skip web-console and java # if no changes in this set we can skip docs @@ -60,11 +60,15 @@ def check_console(file): print("found web-console file change: {}".format(file)) return is_console + +# given a test suite name (travis job name) and set of files in a git diff +# try to determine if the test suite should run or not def check_should_run_suite(suite, diff_files): if suite in always_run_jobs: # you gotta do what you gotta do return True + # is everything in the ignorable list all_ignore = True any_docs = False all_docs = True @@ -94,32 +98,38 @@ def check_should_run_suite(suite, diff_files): return True def failWithUsage(): - sys.stderr.write("usage: check-test-suite.py <test-suite-name>\n") - sys.stderr.write(" e.g., check-test-suite.py docs") + sys.stderr.write("usage: check_test_suite.py <test-suite-name>\n") + sys.stderr.write(" e.g., check_test_suite.py docs") sys.exit(1) -suite_name = "" +if __name__ == '__main__': + suite_name = "" -if len(sys.argv) == 1: - if 'TRAVIS_JOB_NAME' in os.environ: - suite_name = os.environ['TRAVIS_JOB_NAME'] + if len(sys.argv) == 1: + if 'TRAVIS_JOB_NAME' in os.environ: + suite_name = os.environ['TRAVIS_JOB_NAME'] + else: + failWithUsage() + elif len(sys.argv) == 2: + suite_name = sys.argv[1] else: failWithUsage() -elif len(sys.argv) == 2: - suite_name = sys.argv[1] -else: - failWithUsage() -all_changed_files_string = subprocess.check_output("git diff --name-only HEAD~1", shell=True).decode('UTF-8') -all_changed_files = all_changed_files_string.splitlines() -print("Checking if suite '{}' needs to run test on diff:\n{}".format(suite_name, all_changed_files_string)) -needs_run = check_should_run_suite(suite_name, all_changed_files) + all_changed_files_string = subprocess.check_output("git diff --name-only HEAD~1", shell=True).decode('UTF-8') + all_changed_files = all_changed_files_string.splitlines() + print("Checking if suite '{}' needs to run test on diff:\n{}".format(suite_name, all_changed_files_string)) -if needs_run: - print("Changes detected, need to run test suite '{}'".format(suite_name)) - sys.exit(1) + # we should always run all test suites for builds that are not for a pull request + is_pr = False + if 'TRAVIS_PULL_REQUEST' in os.environ and os.environ['TRAVIS_PULL_REQUEST'] != 'false': + is_pr = True + needs_run = not is_pr or check_should_run_suite(suite_name, all_changed_files) + + if needs_run: + print("Changes detected, need to run test suite '{}'".format(suite_name)) + sys.exit(1) -print("No applicable changes detected, can skip test suite '{}'".format(suite_name)) -sys.exit(0) + print("No applicable changes detected, can skip test suite '{}'".format(suite_name)) + sys.exit(0) diff --git a/check_test_suite_test.py b/check_test_suite_test.py new file mode 100755 index 0000000..6c2e4f6 --- /dev/null +++ b/check_test_suite_test.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import check_test_suite + +class CheckTestSuite(unittest.TestCase): + def test_docs(self): + self.assertEqual(False, check_test_suite.check_docs('.travis.yml')) + self.assertEqual(False, check_test_suite.check_docs('check_test_suite_test.py')) + self.assertEqual(True, check_test_suite.check_docs('website/core/Footer.js')) + self.assertEqual(True, check_test_suite.check_docs('docs/ingestion/index.md')) + + for job in check_test_suite.docs_jobs: + self.assertEqual( + True, + check_test_suite.check_should_run_suite(job, ['check_test_suite_test.py', 'docs/ingestion/index.md']) + ) + self.assertEqual( + False, + check_test_suite.check_should_run_suite( + job, + ['check_test_suite_test.py', 'core/src/main/java/org/apache/druid/math/expr/Expr.java'] + ) + ) + + def test_web_console(self): + self.assertEqual(False, check_test_suite.check_console('.travis.yml')) + self.assertEqual(False, check_test_suite.check_console('check_test_suite_test.py')) + self.assertEqual(False, check_test_suite.check_console('website/core/Footer.js')) + self.assertEqual(True, check_test_suite.check_console('web-console/assets/azure.png')) + self.assertEqual(True, check_test_suite.check_console('web-console/src/views/index.ts')) + self.assertEqual(True, check_test_suite.check_console('web-console/unified-console.html')) + + for job in check_test_suite.web_console_jobs: + self.assertEqual( + True, + check_test_suite.check_should_run_suite( + job, + ['check_test_suite_test.py', 'web-console/unified-console.html'] + ) + ) + self.assertEqual( + False, + check_test_suite.check_should_run_suite( + job, + ['check_test_suite_test.py', 'core/src/main/java/org/apache/druid/math/expr/Expr.java'] + ) + ) + + +if __name__ == '__main__': + unittest.main() --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
