This is an automated email from the ASF dual-hosted git repository.
abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new f8f2fe8b7b Skip tests based on files changed in the PR (#14445)
f8f2fe8b7b is described below
commit f8f2fe8b7b06b7b2f08c5d3ae6c99de0a07bcf31
Author: Abhishek Agarwal <[email protected]>
AuthorDate: Thu Jun 22 12:27:23 2023 +0530
Skip tests based on files changed in the PR (#14445)
Our CI system has a lot of tests. And much of this testing is really
unnecessary for most of the PRs. This PR adds some checks so we can skip these
expensive tests when we know they are not necessary.
---
.github/workflows/revised-its.yml | 26 ++++++++++++++
.github/workflows/standard-its.yml | 37 ++++++++++++++++++-
.github/workflows/unit-tests.yml | 42 +++++++++++++++++++---
.../cluster/Common/environment-configs/common.env | 1 +
.../docker/environment-configs/common | 1 +
.../docker/environment-configs/common-ldap | 1 +
.../test-groups/custom-coordinator-duties | 1 +
.../test-groups/prepopulated-data | 1 +
.../test-groups/shuffle-deep-store | 1 +
9 files changed, 106 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/revised-its.yml
b/.github/workflows/revised-its.yml
index e10238b5e1..900b308569 100644
--- a/.github/workflows/revised-its.yml
+++ b/.github/workflows/revised-its.yml
@@ -21,7 +21,30 @@ on:
workflow_dispatch:
jobs:
+ changes:
+ runs-on: ubuntu-latest
+ # Required permissions
+ permissions:
+ pull-requests: read
+ # Set job outputs to values from filter step
+ outputs:
+ # run everything if not a PR
+ core: ${{ steps.filter.outputs.core || github.event_name !=
'pull_request'}}
+ # the common extension in revised ITs is different from the one in
standard ITs
+ common-extensions: ${{ steps.filter.outputs.common-extensions }}
+ steps:
+ - uses: dorny/paths-filter@v2
+ if: github.event_name == 'pull_request'
+ id: filter
+ with:
+ filters: |
+ common-extensions:
+ -
'extension-core/(mysql-metadata-storage|druid-it-tools|druid-lookups-cached-global|druid-histogram|druid-datasketches|druid-parquet-extensions|druid-avro-extensions|druid-protobuf-extensions|druid-orc-extensions|druid-kafka-indexing-service|druid-s3-extensions|druid-multi-stage-query|druid-catalog)/**'
+ core:
+ - '!extension*/**'
+
it:
+ needs: changes
strategy:
fail-fast: false
matrix:
@@ -31,6 +54,7 @@ jobs:
#indexer: [indexer, middleManager]
indexer: [middleManager]
uses: ./.github/workflows/reusable-revised-its.yml
+ if: ${{ needs.changes.outputs.core == 'true' ||
needs.changes.outputs.common-extensions == 'true' }}
with:
build_jdk: ${{ matrix.jdk }}
runtime_jdk: ${{ matrix.jdk }}
@@ -40,7 +64,9 @@ jobs:
mysql_driver: com.mysql.jdbc.Driver
s3-deep-storage-minio:
+ needs: changes
uses: ./.github/workflows/reusable-revised-its.yml
+ if: ${{ needs.changes.outputs.core == 'true' ||
needs.changes.outputs.common-extensions == 'true' }}
with:
build_jdk: 8
runtime_jdk: 11
diff --git a/.github/workflows/standard-its.yml
b/.github/workflows/standard-its.yml
index 155105fab3..2f17222ada 100644
--- a/.github/workflows/standard-its.yml
+++ b/.github/workflows/standard-its.yml
@@ -21,12 +21,35 @@ on:
workflow_dispatch:
jobs:
+ changes:
+ runs-on: ubuntu-latest
+ # Required permissions
+ permissions:
+ pull-requests: read
+ # Set job outputs to values from filter step
+ outputs:
+ # run everything if not a PR
+ core: ${{ steps.filter.outputs.core || github.event_name !=
'pull_request'}}
+ common-extensions: ${{ steps.filter.outputs.common-extensions }}
+ steps:
+ - uses: dorny/paths-filter@v2
+ if: github.event_name == 'pull_request'
+ id: filter
+ with:
+ filters: |
+ common-extensions:
+ -
'extension-core/(mysql-metadata-storage|druid-basic-security|simple-client-sslcontext|druid-testing-tools|druid-lookups-cached-global|druid-histogram|druid-datasketches|druid-parquet-extensions|druid-avro-extensions|druid-protobuf-extensions|druid-orc-extensions|druid-kafka-indexing-service|druid-s3-extensions)/**'
+ core:
+ - '!extension*/**'
+
integration-index-tests-middleManager:
+ needs: changes
strategy:
fail-fast: false
matrix:
testing_group: [batch-index, input-format, input-source,
perfect-rollup-parallel-batch-index, kafka-index, kafka-index-slow,
kafka-transactional-index, kafka-transactional-index-slow, kafka-data-format,
ldap-security, realtime-index, append-ingestion, compaction]
uses: ./.github/workflows/reusable-standard-its.yml
+ if: ${{ needs.changes.outputs.core == 'true' ||
needs.changes.outputs.common-extensions == 'true' }}
with:
build_jdk: 8
runtime_jdk: 8
@@ -35,11 +58,13 @@ jobs:
group: ${{ matrix.testing_group }}
integration-index-tests-indexer:
+ needs: changes
strategy:
fail-fast: false
matrix:
- testing_group: [input-source, perfect-rollup-parallel-batch-index,
kafka-index, kafka-transactional-index, kafka-index-slow,
kafka-transactional-index-slow, kafka-data-format, append-ingestion, compaction]
+ testing_group: [input-source, perfect-rollup-parallel-batch-index,
kafka-index, append-ingestion, compaction]
uses: ./.github/workflows/reusable-standard-its.yml
+ if: ${{ needs.changes.outputs.core == 'true' ||
needs.changes.outputs.common-extensions == 'true' }}
with:
build_jdk: 8
runtime_jdk: 8
@@ -48,11 +73,13 @@ jobs:
group: ${{ matrix.testing_group }}
integration-query-tests-middleManager:
+ needs: changes
strategy:
fail-fast: false
matrix:
testing_group: [query, query-retry, query-error, security,
high-availability]
uses: ./.github/workflows/reusable-standard-its.yml
+ if: ${{ needs.changes.outputs.core == 'true' ||
needs.changes.outputs.common-extensions == 'true' }}
with:
build_jdk: 8
runtime_jdk: 8
@@ -62,11 +89,13 @@ jobs:
group: ${{ matrix.testing_group }}
integration-query-tests-middleManager-mariaDB:
+ needs: changes
strategy:
fail-fast: false
matrix:
jdk: [8, 11]
uses: ./.github/workflows/reusable-standard-its.yml
+ if: ${{ needs.changes.outputs.core == 'true' ||
needs.changes.outputs.common-extensions == 'true' }}
with:
build_jdk: 8
runtime_jdk: ${{ matrix.jdk }}
@@ -77,11 +106,13 @@ jobs:
group: query
integration-shuffle-deep-store-tests:
+ needs: changes
strategy:
fail-fast: false
matrix:
indexer: [indexer, middleManager]
uses: ./.github/workflows/reusable-standard-its.yml
+ if: ${{ needs.changes.outputs.core == 'true' ||
needs.changes.outputs.common-extensions == 'true' }}
with:
build_jdk: 8
runtime_jdk: 8
@@ -91,7 +122,9 @@ jobs:
group: shuffle deep store
integration-custom-coordinator-duties-tests:
+ needs: changes
uses: ./.github/workflows/reusable-standard-its.yml
+ if: ${{ needs.changes.outputs.core == 'true' ||
needs.changes.outputs.common-extensions == 'true' }}
with:
build_jdk: 8
runtime_jdk: 8
@@ -101,7 +134,9 @@ jobs:
group: custom coordinator duties
integration-k8s-leadership-tests:
+ needs: changes
name: (Compile=openjdk8, Run=openjdk8, Cluster Build On K8s)
ITNestedQueryPushDownTest integration test
+ if: ${{ needs.changes.outputs.core == 'true' ||
needs.changes.outputs.common-extensions == 'true' }}
runs-on: ubuntu-22.04
env:
MVN: mvn --no-snapshot-updates
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index f305ba9887..1f75cc4eed 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -31,14 +31,44 @@ on:
description: 'Flag to decide if next tests need to run incase coverage
issue failures'
value: |
${{
- (jobs.indexing_modules_test.result == 'success' ||
fromJson(jobs.indexing_modules_test.outputs.coverage_failure)) &&
- (jobs.processing_modules_test.result == 'success' ||
fromJson(jobs.processing_modules_test.outputs.coverage_failure)) &&
- (jobs.server_modules_test.result == 'success' ||
fromJson(jobs.server_modules_test.outputs.coverage_failure)) &&
- (jobs.other_modules_test.result == 'success' ||
fromJson(jobs.other_modules_test.outputs.coverage_failure))
+ (jobs.indexing_modules_test.result == 'success' ||
jobs.indexing_modules_test.result == 'skipped' ||
+ fromJson(jobs.indexing_modules_test.outputs.coverage_failure)) &&
+ (jobs.processing_modules_test.result == 'success' ||
jobs.processing_modules_test.result == 'skipped' ||
+ fromJson(jobs.processing_modules_test.outputs.coverage_failure)) &&
+ (jobs.server_modules_test.result == 'success' ||
jobs.server_modules_test.result == 'skipped' || fromJson
+ (jobs.server_modules_test.outputs.coverage_failure)) &&
+ (jobs.other_modules_test.result == 'success' ||
jobs.other_modules_test.result == 'skipped' || fromJson(jobs
+ .other_modules_test.outputs.coverage_failure))
}}
jobs:
+ changes:
+ runs-on: ubuntu-latest
+ # Required permissions
+ permissions:
+ pull-requests: read
+ # Set job outputs to values from filter step
+ outputs:
+ kafka: ${{ steps.filter.outputs.kafka }}
+ kinesis: ${{ steps.filter.outputs.kinesis }}
+ # run everything if not a PR
+ core: ${{ steps.filter.outputs.core || github.event_name !=
'pull_request'}}
+ steps:
+ - uses: dorny/paths-filter@v2
+ if: github.event_name == 'pull_request'
+ id: filter
+ with:
+ filters: |
+ core:
+ - '!extension*/**'
+ kafka:
+ - 'extensions-core/kafka-indexing-service/**'
+ kinesis:
+ - 'extensions-core/kinesis-indexing-service/**'
+
indexing_modules_test:
+ needs: changes
+ if: ${{ needs.changes.outputs.core == 'true' ||
needs.changes.outputs.kafka == 'true' || needs.changes.outputs.kinesis ==
'true'}}
uses: ./.github/workflows/reusable-unit-tests.yml
with:
jdk: ${{ inputs.jdk }}
@@ -47,6 +77,8 @@ jobs:
maven_projects:
'indexing-hadoop,indexing-service,extensions-core/kafka-indexing-service,extensions-core/kinesis-indexing-service'
processing_modules_test:
+ needs: changes
+ if: ${{ needs.changes.outputs.core == 'true' }}
uses: ./.github/workflows/reusable-unit-tests.yml
with:
jdk: ${{ inputs.jdk }}
@@ -55,6 +87,8 @@ jobs:
maven_projects: 'processing'
server_modules_test:
+ needs: changes
+ if: ${{ needs.changes.outputs.core == 'true' }}
uses: ./.github/workflows/reusable-unit-tests.yml
with:
jdk: ${{ inputs.jdk }}
diff --git
a/integration-tests-ex/cases/cluster/Common/environment-configs/common.env
b/integration-tests-ex/cases/cluster/Common/environment-configs/common.env
index 8b58cd949e..6f9dd9ba94 100644
--- a/integration-tests-ex/cases/cluster/Common/environment-configs/common.env
+++ b/integration-tests-ex/cases/cluster/Common/environment-configs/common.env
@@ -49,6 +49,7 @@ DRUID_INSTANCE=
# variables: druid_standard_loadList defined here, and druid_test_loadList,
defined
# in a docker-compose.yaml file, for any test-specific extensions.
# See compose.md for more details.
+# If you are making a change in load list below, make the necessary changes in
github actions too
druid_standard_loadList=mysql-metadata-storage,druid-it-tools,druid-lookups-cached-global,druid-histogram,druid-datasketches,druid-parquet-extensions,druid-avro-extensions,druid-protobuf-extensions,druid-orc-extensions,druid-kafka-indexing-service,druid-s3-extensions,druid-multi-stage-query,druid-catalog
# Location of Hadoop dependencies provided at runtime in the shared directory.
diff --git a/integration-tests/docker/environment-configs/common
b/integration-tests/docker/environment-configs/common
index a1a294bb1b..1aec119e8d 100644
--- a/integration-tests/docker/environment-configs/common
+++ b/integration-tests/docker/environment-configs/common
@@ -26,6 +26,7 @@ COMMON_DRUID_JAVA_OPTS=-Duser.timezone=UTC
-Dfile.encoding=UTF-8 -Dlog4j.configu
DRUID_DEP_LIB_DIR=/shared/hadoop_xml:/shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
# Druid configs
+# If you are making a change in load list below, make the necessary changes in
github actions too
druid_extensions_loadList=["mysql-metadata-storage","druid-basic-security","simple-client-sslcontext","druid-testing-tools","druid-lookups-cached-global","druid-histogram","druid-datasketches","druid-parquet-extensions","druid-avro-extensions","druid-protobuf-extensions","druid-orc-extensions","druid-kafka-indexing-service","druid-s3-extensions"]
druid_startup_logging_logProperties=true
druid_extensions_directory=/shared/docker/extensions
diff --git a/integration-tests/docker/environment-configs/common-ldap
b/integration-tests/docker/environment-configs/common-ldap
index b49aa1a45c..261752d8ba 100644
--- a/integration-tests/docker/environment-configs/common-ldap
+++ b/integration-tests/docker/environment-configs/common-ldap
@@ -27,6 +27,7 @@ COMMON_DRUID_JAVA_OPTS=-Duser.timezone=UTC
-Dfile.encoding=UTF-8 -Dlog4j.configu
DRUID_DEP_LIB_DIR=/shared/hadoop_xml:/shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
# Druid configs
+# If you are making a change in load list below, make the necessary changes in
github actions too
druid_extensions_loadList=["mysql-metadata-storage","druid-s3-extensions","druid-basic-security","simple-client-sslcontext","druid-testing-tools","druid-lookups-cached-global","druid-histogram","druid-datasketches"]
druid_extensions_directory=/shared/docker/extensions
druid_auth_authenticator_ldap_authorizerName=ldapauth
diff --git
a/integration-tests/docker/environment-configs/test-groups/custom-coordinator-duties
b/integration-tests/docker/environment-configs/test-groups/custom-coordinator-duties
index 0fd17a4961..5c75c4197d 100644
---
a/integration-tests/docker/environment-configs/test-groups/custom-coordinator-duties
+++
b/integration-tests/docker/environment-configs/test-groups/custom-coordinator-duties
@@ -17,6 +17,7 @@
# under the License.
#
+# If you are making a change in load list below, make the necessary changes in
github actions too
druid_extensions_loadList=["druid-kafka-indexing-service","mysql-metadata-storage","druid-s3-extensions","druid-basic-security","simple-client-sslcontext","druid-testing-tools","druid-lookups-cached-global","druid-histogram","druid-datasketches"]
druid_coordinator_period_metadataStoreManagementPeriod=PT1H
diff --git
a/integration-tests/docker/environment-configs/test-groups/prepopulated-data
b/integration-tests/docker/environment-configs/test-groups/prepopulated-data
index acce51db6d..deb59dfe40 100644
--- a/integration-tests/docker/environment-configs/test-groups/prepopulated-data
+++ b/integration-tests/docker/environment-configs/test-groups/prepopulated-data
@@ -19,6 +19,7 @@
AWS_REGION=us-east-1
+# If you are making a change in load list below, make the necessary changes in
github actions too
druid_extensions_loadList=["mysql-metadata-storage","druid-s3-extensions","druid-basic-security","simple-client-sslcontext","druid-testing-tools","druid-lookups-cached-global","druid-histogram","druid-datasketches","druid-integration-tests"]
# Setting s3 credentials and region to use pre-populated data for testing.
diff --git
a/integration-tests/docker/environment-configs/test-groups/shuffle-deep-store
b/integration-tests/docker/environment-configs/test-groups/shuffle-deep-store
index 70a6a65940..731d7254f8 100644
---
a/integration-tests/docker/environment-configs/test-groups/shuffle-deep-store
+++
b/integration-tests/docker/environment-configs/test-groups/shuffle-deep-store
@@ -19,5 +19,6 @@
# Test with deep storage as intermediate location to store shuffle data
# Local deep storage will be used here
+# If you are making a change in load list below, make the necessary changes in
github actions too
druid_extensions_loadList=["mysql-metadata-storage","druid-basic-security","simple-client-sslcontext","druid-testing-tools","druid-lookups-cached-global","druid-histogram","druid-datasketches"]
druid_processing_intermediaryData_storage_type=deepstore
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
