This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new ecbc1c7b7e0 Run addition jdks in the main ci job (#17876)
ecbc1c7b7e0 is described below
commit ecbc1c7b7e0e102d48cab460f5ff0c16b0d90c12
Author: Zoltan Haindrich <[email protected]>
AuthorDate: Mon Apr 7 11:44:13 2025 +0200
Run addition jdks in the main ci job (#17876)
* moves back the execution to be under ci.yml
* enables to run all jdk tests if the PR has the label: `tests:all-jdk`
* run all-jdk tests on all branches
* minor cleanup: changed the labels of the jobs; rebalanced the execution
* added a script which helps to get the inputs for rebalancing (it's also
called after jacoco is executed - as that job already has the necessery details)
---
.github/scripts/create-jacoco-coverage-report.sh | 4 ++
.github/scripts/test_balancing_calc | 38 +++++++++++
.github/workflows/ci.yml | 38 +++++++----
.github/workflows/cron-job-its.yml | 4 ++
.github/workflows/cron-job-unit-tests.yml | 79 ----------------------
.../unit-and-integration-tests-unified.yml | 8 +--
.github/workflows/worker.yml | 4 +-
7 files changed, 74 insertions(+), 101 deletions(-)
diff --git a/.github/scripts/create-jacoco-coverage-report.sh
b/.github/scripts/create-jacoco-coverage-report.sh
index b5af7765013..489dfda2834 100755
--- a/.github/scripts/create-jacoco-coverage-report.sh
+++ b/.github/scripts/create-jacoco-coverage-report.sh
@@ -68,3 +68,7 @@ then
-- ||
{ printf "\n\n****FAILED****\nDiff code coverage check failed. To view
coverage report, run 'mvn clean test jacoco:report' and open
'target/site/jacoco/index.html'\nFor more details on how to run code coverage
locally, follow instructions here -
https://github.com/apache/druid/blob/master/dev/code-review/code-coverage.md#running-code-coverage-locally\n\n";
&& exit 1; }
fi
+
+
+echo " * test balancing details"
+.github/scripts/test_balancing_calc
diff --git a/.github/scripts/test_balancing_calc
b/.github/scripts/test_balancing_calc
new file mode 100755
index 00000000000..f619278b106
--- /dev/null
+++ b/.github/scripts/test_balancing_calc
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#--------------------------------------------------------------------
+
+# Helps to calculate new balancing
+# on the output a simple knapsack should be done...right now that's manual
+
+set -e
+
+find . -name 'TEST*xml' |
+# <testsuite ... name="org.apache.druid.query.policy.NoRestrictionPolicyTest"
time="0.071" ...>
+xargs sed -nr
'/^<testsuite/s/.*name=\"([^\"]+)\".*time=\"([^\"]+)\".*/\1\t\2/p' |
+# org.apache.druid.server.RequestLogLineTest 0.052
+sed -r 's/[^\t]+\.(.)[^\t.]+\t/\1\t/' > test_times
+
+for c in {A..Z} ;do
+ echo -n "$c ";
+ (echo 0;grep "^$c" test_times| cut -f 2)|paste -s -d + -|bc
+done | sort -k 2 -nr
+# final output is a table like
+# <letter> <sumTime>
+# C 88
+
+
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 191b53ac2fb..70da61e9449 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -17,28 +17,23 @@ name: "CI"
on:
workflow_call:
- inputs:
- jdk:
- required: false
- type: string
- default: '17'
jobs:
run-unit-tests:
- name: "unit tests"
+ name: "unit tests(main)"
strategy:
fail-fast: false
matrix:
- Dtest: [ "A*,F*,S*", "B*,D*,L*,T*", "C*,O*", "E*,N*,Q*", "G*,R*,U*",
"H*,I*,J*", "K*,P*,V*,W*,X*,Y*,Z*", "M*"]
+ pattern: [ "A*,G*,R*", "B*,O*,S*,X*,Y*,Z*", "C*,E*", "D*,J*,K*",
"F*,H*,U*", "I*,N*,T*", "L*,Q*,W*", "M*,P*,V*"]
uses: ./.github/workflows/worker.yml
with:
- script: .github/scripts/run-unit-tests.sh -Dtest='${{ matrix.Dtest }}'
-Dmaven.test.failure.ignore=true
- jdk: ${{ inputs.jdk }}
+ script: .github/scripts/run-unit-tests.sh -Dtest='${{ matrix.pattern }}'
-Dmaven.test.failure.ignore=true
+ jdk: 17
artifact_prefix: "unit-test-reports"
- key: ${{ matrix.Dtest }}
+ key: "test-jdk17-[${{ matrix.pattern }}]"
- reporting-unit-test-failures:
- name: "report-unit-test-failures"
+ test-report:
+ name: "test-report"
needs: run-unit-tests
runs-on: ubuntu-latest
steps:
@@ -61,11 +56,26 @@ jobs:
truncate_stack_traces: false
reporting-jacoco-coverage-failures:
- name: "report-jacoco-coverage-failures"
+ name: "coverage-jacoco"
needs: run-unit-tests
uses: ./.github/workflows/worker.yml
if: ${{ !contains( github.event.pull_request.labels.*.name, 'jacoco:skip')
}}
with:
script: .github/scripts/create-jacoco-coverage-report.sh
artifacts_to_download: "unit-test-reports-*"
- key: "jacoco-coverage-report"
+ key: "coverage-jacoco"
+
+ run-unit-tests-all-jdk:
+ needs: test-report
+ name: "unit tests(all jdk)"
+ if: ${{ !cancelled() && ( contains(
github.event.pull_request.labels.*.name, 'tests:all-jdk') || github.event_name
== 'push' ) }}
+ strategy:
+ fail-fast: false
+ matrix:
+ jdk: [ "11", "21.0.4" ]
+ pattern: [ "A*,G*,R*", "B*,O*,S*,X*,Y*,Z*", "C*,E*", "D*,J*,K*",
"F*,H*,U*", "I*,N*,T*", "L*,Q*,W*", "M*,P*,V*"]
+ uses: ./.github/workflows/worker.yml
+ with:
+ script: .github/scripts/run-unit-tests.sh -Dtest='${{ matrix.pattern }}'
-fae
+ jdk: ${{ matrix.jdk }}
+ key: "test-jdk${{ matrix.jdk }}-[${{ matrix.pattern }}]"
diff --git a/.github/workflows/cron-job-its.yml
b/.github/workflows/cron-job-its.yml
index ddf26c019ee..6084577b9d2 100644
--- a/.github/workflows/cron-job-its.yml
+++ b/.github/workflows/cron-job-its.yml
@@ -24,6 +24,10 @@ on:
- master
- '[0-9]+.[0-9]+.[0-9]+' # release branches
- '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches
+ push:
+ branches:
+ - '[0-9]+.[0-9]+.[0-9]+' # release branches
+ - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches
jobs:
build:
diff --git a/.github/workflows/cron-job-unit-tests.yml
b/.github/workflows/cron-job-unit-tests.yml
deleted file mode 100644
index cb9f1117bab..00000000000
--- a/.github/workflows/cron-job-unit-tests.yml
+++ /dev/null
@@ -1,79 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "Cron Job Unit Tests"
-on:
- schedule: # Runs by default on master branch
- - cron: '0 3 * * 6' # Runs every Saturday at 03:00 AM UTC
- pull_request:
- branches:
- - '[0-9]+.[0-9]+.[0-9]+' # release branches
- - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches
-
-jobs:
- run-unit-tests:
- name: "unit tests"
- if: github.repository == 'apache/druid'
- strategy:
- fail-fast: false
- matrix:
- jdk: [ '11', '21.0.4' ]
- Dtest: [ "A*,F*,S*", "B*,D*,L*,T*", "C*,O*", "E*,N*,Q*", "G*,R*,U*",
"H*,I*,J*", "K*,P*,V*,W*,X*,Y*,Z*", "M*"]
- uses: ./.github/workflows/worker.yml
- with:
- script: .github/scripts/run-unit-tests.sh -Dtest='${{ matrix.Dtest }}'
-Dmaven.test.failure.ignore=true
- jdk: ${{ matrix.jdk }}
- artifact_prefix: "unit-test-reports-${{ matrix.jdk }}"
- key: ${{ matrix.Dtest }}
-
- reporting-unit-test-failures:
- name: "report-unit-test-failures-${{ matrix.jdk }}"
- needs: run-unit-tests
- runs-on: ubuntu-latest
- strategy:
- fail-fast: false
- matrix:
- jdk: [ '11', '21.0.4' ]
- steps:
- - name: Download reports for all unit test jobs
- uses: actions/download-artifact@v4
- with:
- pattern: "unit-test-reports-${{ matrix.jdk }}-*"
- path: target/surefire-reports
-
- - name: Publish Test Report
- uses: mikepenz/action-junit-report@v5
- with:
- check_name: "Unit Test Report (JDK ${{ matrix.jdk }})"
- report_paths: '**/target/surefire-reports/TEST-*.xml'
- detailed_summary: true
- flaky_summary: true
- annotate_only: true
- fail_on_failure: true
- check_retries: true
- truncate_stack_traces: false
-
- reporting-jacoco-coverage-failures:
- name: "report-jacoco-coverage-failures-${{ matrix.jdk }}"
- needs: run-unit-tests
- uses: ./.github/workflows/worker.yml
- strategy:
- fail-fast: false
- matrix:
- jdk: [ '11', '21.0.4' ]
- with:
- script: .github/scripts/create-jacoco-coverage-report.sh
- artifacts_to_download: "unit-test-reports-${{ matrix.jdk }}-*"
- key: "jacoco-coverage-report"
diff --git a/.github/workflows/unit-and-integration-tests-unified.yml
b/.github/workflows/unit-and-integration-tests-unified.yml
index e15bdaa278e..131029f9fdd 100644
--- a/.github/workflows/unit-and-integration-tests-unified.yml
+++ b/.github/workflows/unit-and-integration-tests-unified.yml
@@ -192,20 +192,16 @@ jobs:
run: echo "::set-output name=isApproved::true"
unit-tests-unapproved:
- name: "unit tests - PR unapproved - (jdk17)"
+ name: "unit tests - PR unapproved"
uses: ./.github/workflows/ci.yml
needs: [check-approval]
if: ${{ needs.check-approval.outputs.approved != 'true' }}
- with:
- jdk: 17
unit-tests-approved:
- name: "unit tests - PR approved - (jdk17)"
+ name: "unit tests - PR approved"
uses: ./.github/workflows/ci.yml
needs: [check-approval]
if: ${{ needs.check-approval.outputs.approved == 'true' }}
- with:
- jdk: 17
standard-its:
needs: [build, unit-tests-unapproved, check-approval]
diff --git a/.github/workflows/worker.yml b/.github/workflows/worker.yml
index d25cf0cdba0..7c0b7842e4e 100644
--- a/.github/workflows/worker.yml
+++ b/.github/workflows/worker.yml
@@ -29,7 +29,7 @@ on:
key:
required: false
type: string
- description: "Key for the matrix"
+ description: "Identifier for job"
artifact_prefix:
required: false
type: string
@@ -44,7 +44,7 @@ env:
jobs:
execute:
- name: "Execute (${{ inputs.key }})"
+ name: "${{ inputs.key }}"
runs-on: ubuntu-latest
steps:
- name: Checkout repository
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
