zhouyuan commented on code in PR #13421: URL: https://github.com/apache/gluten/pull/13421#discussion_r4366448394
########## .github/workflows/iceberg_spark_ut.yml: ########## @@ -15,43 +15,14 @@ name: Iceberg Spark UT (Gluten) +# Pull requests run this from velox_backend_x86.yml, which passes the static native libraries +# it already built for the head revision. Manual runs build them here. on: - pull_request: - paths: - - '.github/workflows/iceberg_spark_ut.yml' - - '.github/workflows/iceberg_upstream_revision.yml' - - '.github/workflows/velox_native_build.yml' - - '.github/workflows/util/iceberg-upstream/**' - - 'backends-velox/src-iceberg*/**' - - 'gluten-iceberg/**' - - 'gluten-ut/src-iceberg-upstream/**' - # Shared production code and dependencies used by Iceberg on Spark 3.5. - - 'pom.xml' - - 'backends-velox/pom.xml' - - 'backends-velox/src/main/**' - - 'gluten-core/pom.xml' - - 'gluten-core/src/main/**' - - 'gluten-substrait/pom.xml' - - 'gluten-substrait/src/main/**' - - 'gluten-arrow/pom.xml' - - 'gluten-arrow/src/main/**' - - 'shims/pom.xml' - - 'shims/common/pom.xml' - - 'shims/common/src/main/**' - - 'shims/spark35/pom.xml' - - 'shims/spark35/src/main/**' - - 'build/mvn' - - 'dev/gluten-build-info.sh' - # Native code and the build inputs used by velox_native_build.yml. - - 'cpp/**' - - 'ep/build-velox/**' - - 'dev/ci-velox-buildstatic-centos-8.sh' - - 'dev/builddeps-veloxbe.sh' - - 'dev/build-helper-functions.sh' - - 'dev/vcpkg/**' - # Documentation-only changes do not affect this runtime. - - '!**/*.md' - - '!**/*.rst' + workflow_call: Review Comment: I'm afraid this will make the CI job a bit too long to wait - for the iceberg CI it will need to wait for 1hour and run those tests ########## .github/workflows/iceberg_spark_ut.yml: ########## @@ -15,43 +15,14 @@ name: Iceberg Spark UT (Gluten) +# Pull requests run this from velox_backend_x86.yml, which passes the static native libraries +# it already built for the head revision. Manual runs build them here. on: - pull_request: - paths: - - '.github/workflows/iceberg_spark_ut.yml' - - '.github/workflows/iceberg_upstream_revision.yml' - - '.github/workflows/velox_native_build.yml' - - '.github/workflows/util/iceberg-upstream/**' - - 'backends-velox/src-iceberg*/**' - - 'gluten-iceberg/**' - - 'gluten-ut/src-iceberg-upstream/**' - # Shared production code and dependencies used by Iceberg on Spark 3.5. - - 'pom.xml' - - 'backends-velox/pom.xml' - - 'backends-velox/src/main/**' - - 'gluten-core/pom.xml' - - 'gluten-core/src/main/**' - - 'gluten-substrait/pom.xml' - - 'gluten-substrait/src/main/**' - - 'gluten-arrow/pom.xml' - - 'gluten-arrow/src/main/**' - - 'shims/pom.xml' - - 'shims/common/pom.xml' - - 'shims/common/src/main/**' - - 'shims/spark35/pom.xml' - - 'shims/spark35/src/main/**' Review Comment: for now iceberg only runs against spark-3.5, it's a trade off to reduce CI usage ########## .github/workflows/velox_kafka.yml: ########## @@ -1,161 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Velox Kafka - -on: - pull_request: - paths: - - '.github/workflows/velox_kafka.yml' - - 'pom.xml' - - 'backends-velox/**' - - 'gluten-kafka/**' - - 'gluten-core/**' - - 'gluten-substrait/**' - - 'gluten-ut/**' - - 'shims/**' - - 'cpp/**' - - 'ep/build-velox/**' - - 'dev/**' - - 'build/mvn' - workflow_dispatch: - -permissions: - contents: read - actions: read - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - CCACHE_DIR: ${{ github.workspace }}/.ccache - PROMPT_ALWAYS_RESPOND: 'y' - -jobs: - build-native: - runs-on: ubuntu-22.04 - container: apache/gluten:vcpkg-almalinux-8-gcc13 - timeout-minutes: 180 - steps: - - uses: actions/checkout@v7 - - name: Install build tools - run: | - dnf install -y jq bison tzdata - dnf module -y install python39 - alternatives --set python3 /usr/bin/python3.9 - dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo - dnf install -y gh --setopt=gpgcheck=0 - - name: Get Ccache from Apache Stash - id: ccache-stash - uses: apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-almalinux8-release-default-${{ hashFiles('ep/build-velox/src/**') }} - - name: Get Ccache from actions/cache (fallback on Stash miss) - if: steps.ccache-stash.outputs.stash-hit != 'true' - uses: actions/cache/restore@v4 - with: - path: ${{ env.CCACHE_DIR }} - key: ccache-almalinux8-release-default-${{ github.sha }} - restore-keys: | - ccache-almalinux8-release-default - - name: Build Kafka-enabled native library and tests - shell: bash - run: | - source /opt/rh/gcc-toolset-13/enable - export NUM_THREADS=2 CCACHE_MAXSIZE=1G - ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --enable_kafka=ON \ - --build_arrow=OFF --build_tests=ON --build_benchmarks=OFF --build_examples=OFF \ - --enable_s3=OFF --enable_gcs=OFF --enable_hdfs=OFF --enable_abfs=OFF - - name: Run native Kafka connector tests - run: ./cpp/build/velox/tests/velox_kafka_connector_test - - name: Run Kafka and file plan conversion tests - run: | - ./cpp/build/velox/tests/velox_plan_conversion_test \ - --gtest_filter='Substrait2VeloxPlanConversionTest.serializedFileSplit:Substrait2VeloxPlanConversionTest.kafka*:Substrait2VeloxPlanConversionTest.mixedKafkaAndFileSplits:Substrait2VeloxPlanValidatorTest.kafkaReadRequiresNativeSupport' - - uses: actions/upload-artifact@v7 - with: - name: velox-kafka-native-${{ github.sha }} - path: cpp/build/releases/ - if-no-files-found: error - retention-days: 1 - - spark35: - needs: build-native - runs-on: ubuntu-22.04 - timeout-minutes: 60 - env: - KAFKA_BOOTSTRAP_SERVERS: localhost:9092 - SPARK_LOCAL_IP: 127.0.0.1 - services: - kafka: - image: apache/kafka:3.9.1 - ports: - - 9092:9092 - env: - CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk - KAFKA_NODE_ID: 1 - KAFKA_PROCESS_ROLES: broker,controller - KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT - KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER - KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - options: >- - --health-cmd "/opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --list" - --health-interval 10s - --health-timeout 10s - --health-retries 12 - steps: - - uses: actions/checkout@v7 - - name: Setup java - run: | - sudo apt-get update - sudo apt-get install -y openjdk-17-jdk - echo "JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64" >> $GITHUB_ENV - echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH - - uses: actions/download-artifact@v8 - with: - name: velox-kafka-native-${{ github.sha }} - path: cpp/build/releases/ - - name: Install Spark 3.5 modules with Kafka support - run: | - ./build/mvn -B -ntp install -Pbackends-velox -Pspark-3.5 -Pscala-2.12 \ - -Pjava-17 -Pkafka -Pspark-ut -Pfast-build -DskipTests - # TODO: Re-enable when VeloxKafkaScanValidationSuite is added. Review Comment: This will be recovered once https://github.com/apache/gluten/pull/13237 landed ########## .github/workflows/velox_backend_enhanced.yml: ########## @@ -15,29 +15,10 @@ name: Velox Backend (Enhanced) +# Paused on pull requests to save CI capacity: the enhanced-features build is a separate +# static Velox build that no other job reuses. Run it on demand when needed. on: - pull_request: - paths: - - '.github/workflows/velox_backend_enhanced.yml' - - '.github/workflows/util/parallel-ut/**' - - 'pom.xml' - - 'backends-velox/**' - - 'gluten-uniffle/**' - - 'gluten-celeborn/**' - - 'gluten-core/**' - - 'gluten-substrait/**' - - 'gluten-arrow/**' - - 'gluten-delta/**' - - 'gluten-iceberg/**' - - 'gluten-hudi/**' - - 'gluten-ut/**' - - 'package/**' - - 'shims/**' - - 'tools/gluten-it/**' - - 'ep/build-velox/**' - - 'cpp/**' - - 'dev/**' - - 'build/mvn' + workflow_dispatch: Review Comment: For transparency there is no pending upstream patches in enhanced branch now. We kept it here just in case if potential urgent patches required ########## .github/workflows/velox_kafka.yml: ########## @@ -1,161 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Velox Kafka Review Comment: The Kafka CI is very short, only occupy 2.4% in last 7 days -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
