github-advanced-security[bot] commented on code in PR #4884:
URL: https://github.com/apache/datafusion-comet/pull/4884#discussion_r3595714801
##########
.github/workflows/pr_build_linux.yml:
##########
@@ -582,3 +582,131 @@
SPARK_TPCDS_JOIN_CONF: |
spark.sql.autoBroadcastJoinThreshold=-1
spark.sql.join.forceApplyShuffledHashJoin=true
+
+ # Uniffle integration test - verifies Comet shuffle against a local Uniffle
cluster
+ uniffle-integration-test:
+ needs: build-native
+ name: Uniffle Integration Test
+ runs-on: ubuntu-24.04
+ container:
+ image: amd64/rust
+ env:
+ JAVA_TOOL_OPTIONS: --add-exports=java.base/sun.nio.ch=ALL-UNNAMED
--add-exports=java.base/sun.util.calendar=ALL-UNNAMED
--add-opens=java.base/java.nio=ALL-UNNAMED
--add-opens=java.base/java.lang=ALL-UNNAMED
+ UNIFFLE_VERSION: 0.10.0
+ HADOOP_VERSION: 2.10.2
+ steps:
+ - uses: actions/checkout@v7
+
+ - name: Setup Rust & Java toolchain
+ uses: ./.github/actions/setup-builder
+ with:
+ rust-version: ${{ env.RUST_VERSION }}
+ jdk-version: 17
+
+ - name: Download native library
+ uses: actions/download-artifact@v8
+ with:
+ name: native-lib-linux
+ path: native/target/release/
+
+ - name: Cache Maven dependencies
+ uses: actions/cache@v6
+ with:
+ path: |
+ ~/.m2/repository
+ /root/.m2/repository
+ key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-java-maven-
+
+ - name: Cache TPC-H data
+ id: cache-uniffle-tpch
+ uses: actions/cache@v6
+ with:
+ path: ./tpch
+ key: tpch-${{ hashFiles('.github/workflows/pr_build_linux.yml') }}
+
+ - name: Cache TPC-DS data
+ id: cache-uniffle-tpcds
+ uses: actions/cache@v6
+ with:
+ path: ./tpcds-sf-1
+ key: tpcds-${{ hashFiles('.github/workflows/pr_build_linux.yml') }}
+
+ - name: Install Uniffle
+ run: |
+ wget
"https://www.apache.org/dyn/closer.lua/uniffle/${UNIFFLE_VERSION}/apache-uniffle-${UNIFFLE_VERSION}-bin.tar.gz?action=download";
\
+ -O "/opt/apache-uniffle-${UNIFFLE_VERSION}-bin.tar.gz"
+ wget
"https://www.apache.org/dyn/closer.lua/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz?action=download";
\
+ -O "/opt/hadoop-${HADOOP_VERSION}.tar.gz"
+ mkdir /opt/uniffle
+ tar xzf "/opt/apache-uniffle-${UNIFFLE_VERSION}-bin.tar.gz" \
+ -C /opt/uniffle --strip-components=1
+ tar xzf "/opt/hadoop-${HADOOP_VERSION}.tar.gz" -C /opt/
+ mkdir /opt/uniffle/shuffle_data
+ printf 'XMX_SIZE=16g\nHADOOP_HOME=/opt/hadoop-%s\n'
"${HADOOP_VERSION}" \
+ > /opt/uniffle/conf/rss-env.sh
+ printf 'rss.coordinator.shuffle.nodes.max 1\nrss.rpc.server.port
19999\n' \
+ > /opt/uniffle/conf/coordinator.conf
+ printf '%s\n' \
+ 'rss.server.app.expired.withoutHeartbeat 7200000' \
+ 'rss.server.heartbeat.delay 3000' \
+ 'rss.rpc.server.port 19997' \
+ 'rss.rpc.server.type GRPC_NETTY' \
+ 'rss.jetty.http.port 19996' \
+ 'rss.server.netty.port 19995' \
+ 'rss.storage.basePath /opt/uniffle/shuffle_data' \
+ 'rss.storage.type MEMORY_LOCALFILE' \
+ 'rss.coordinator.quorum localhost:19999' \
+ 'rss.server.flush.thread.alive 10' \
+ 'rss.server.single.buffer.flush.threshold 64m' \
+ > /opt/uniffle/conf/server.conf
+ cd /opt/uniffle
+ bash ./bin/start-coordinator.sh
+ bash ./bin/start-shuffle-server.sh
+
+ - name: Build project
+ run: |
+ ./mvnw -B -Prelease install -DskipTests -Pspark-3.5,uniffle
+
+ - name: Generate TPC-H data (SF=1)
+ if: steps.cache-uniffle-tpch.outputs.cache-hit != 'true'
+ run: |
+ cd spark && MAVEN_OPTS='-Xmx20g' ../mvnw -B -Prelease -Pspark-3.5
exec:java -Dexec.mainClass="org.apache.spark.sql.GenTPCHData"
-Dexec.classpathScope="test" -Dexec.cleanupDaemonThreads="false"
-Dexec.args="--location `pwd`/.. --scaleFactor 1 --numPartitions 1 --overwrite"
+
+ - name: Checkout tpcds-kit
+ if: steps.cache-uniffle-tpcds.outputs.cache-hit != 'true'
+ uses: actions/checkout@v7
+ with:
+ repository: databricks/tpcds-kit
+ path: ./tpcds-kit
+
+ - name: Build tpcds-kit
+ if: steps.cache-uniffle-tpcds.outputs.cache-hit != 'true'
+ run: |
+ apt-get update && apt-get install -y yacc bison flex gcc-12 g++-12
+ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 120
--slave /usr/bin/g++ g++ /usr/bin/g++-12
+ cd tpcds-kit/tools && make OS=LINUX
+
+ - name: Generate TPC-DS data (SF=1)
+ if: steps.cache-uniffle-tpcds.outputs.cache-hit != 'true'
+ run: |
+ cd spark && MAVEN_OPTS='-Xmx20g' ../mvnw -B -Prelease -Pspark-3.5
exec:java -Dexec.mainClass="org.apache.spark.sql.GenTPCDSData"
-Dexec.classpathScope="test" -Dexec.cleanupDaemonThreads="false"
-Dexec.args="--dsdgenDir `pwd`/../tpcds-kit/tools --location
`pwd`/../tpcds-sf-1 --scaleFactor 1 --numPartitions 1"
+
+ - name: Run TPC-H queries with Uniffle
+ run: |
+ SPARK_HOME=`pwd` SPARK_TPCH_DATA=`pwd`/tpch/sf1_parquet ./mvnw -B
-Prelease -Pspark-3.5,uniffle,uniffle-comet-test -Dtest=none
-Dsuites=org.apache.spark.sql.CometTPCHQuerySuite test
+ env:
+ COMET_SHUFFLE_MANAGER: uniffle
+ SPARK_TPCH_JOIN_CONF: |
+ spark.sql.autoBroadcastJoinThreshold=-1
+ spark.sql.join.preferSortMergeJoin=true
+
+ - name: Run TPC-DS queries with Uniffle
+ run: |
+ SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -B
-Prelease -Pspark-3.5,uniffle,uniffle-comet-test -Dtest=none
-Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test
+ env:
+ COMET_SHUFFLE_MANAGER: uniffle
+ SPARK_TPCDS_JOIN_CONF: |
+ spark.sql.autoBroadcastJoinThreshold=-1
+ spark.sql.join.preferSortMergeJoin=true
Review Comment:
## CodeQL / Workflow does not contain permissions
Actions job or workflow does not limit the permissions of the GITHUB_TOKEN.
Consider setting an explicit permissions block, using the following as a
minimal starting point: {{contents: read}}
[Show more
details](https://github.com/apache/datafusion-comet/security/code-scanning/124)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
