This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch branch-1.10
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/branch-1.10 by this push:
new 31470eca9d [KYUUBI #7205] Use Apache closer.lua syntax for
kyuubi-download to obtain engine archives
31470eca9d is described below
commit 31470eca9d587b8c06c09019bb102f8f998a6c2a
Author: Cheng Pan <[email protected]>
AuthorDate: Mon Sep 22 15:15:09 2025 +0800
[KYUUBI #7205] Use Apache closer.lua syntax for kyuubi-download to obtain
engine archives
Speed up download Spark, Flink, and Hive tarballs, for the latest versions.
https://infra.apache.org/release-download-pages.html#download-scripts
Observe GHA.
No.
Closes #7205 from pan3793/closer-lua.
Closes #7205
10ffa3441 [Cheng Pan] Use Apache closer.lua syntax for kyuubi-download to
obtain engine archives
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
(cherry picked from commit 28bba27df8811d5be1187bc2f262d30f4737d0fe)
Signed-off-by: Cheng Pan <[email protected]>
---
.github/workflows/master.yml | 18 +++++++++---------
docs/contributing/code/building.md | 2 +-
externals/kyuubi-download/pom.xml | 9 ++++++---
pom.xml | 8 +++++++-
4 files changed, 23 insertions(+), 14 deletions(-)
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index dd03ddea7d..10ab85dbd3 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -62,22 +62,22 @@ jobs:
comment: 'normal'
- java: 8
spark: '3.5'
- spark-archive:
'-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.4
-Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz -Pzookeeper-3.6'
+ spark-archive:
'-Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-3.2.4
-Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz -Pzookeeper-3.6'
exclude-tags:
'-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
comment: 'verify-on-spark-3.2-binary'
- java: 8
spark: '3.5'
- spark-archive:
'-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.3.3
-Dspark.archive.name=spark-3.3.3-bin-hadoop3.tgz -Pzookeeper-3.6'
+ spark-archive:
'-Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-3.3.3
-Dspark.archive.name=spark-3.3.3-bin-hadoop3.tgz -Pzookeeper-3.6'
exclude-tags:
'-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
comment: 'verify-on-spark-3.3-binary'
- java: 8
spark: '3.5'
- spark-archive:
'-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.4.3
-Dspark.archive.name=spark-3.4.3-bin-hadoop3.tgz -Pzookeeper-3.6'
+ spark-archive:
'-Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-3.4.3
-Dspark.archive.name=spark-3.4.3-bin-hadoop3.tgz -Pzookeeper-3.6'
exclude-tags:
'-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
comment: 'verify-on-spark-3.4-binary'
- java: 17
spark: '3.5'
- spark-archive: '-Pscala-2.13
-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-4.0.1
-Dspark.archive.name=spark-4.0.1-bin-hadoop3.tgz'
+ spark-archive: '-Pscala-2.13
-Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-4.0.1
-Dspark.archive.name=spark-4.0.1-bin-hadoop3.tgz'
exclude-tags:
'-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
comment: 'verify-on-spark-4.0-binary'
env:
@@ -254,15 +254,15 @@ jobs:
include:
- java: 8
flink: '1.20'
- flink-archive:
'-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.17.2
-Dflink.archive.name=flink-1.17.2-bin-scala_2.12.tgz'
+ flink-archive:
'-Dflink.archive.mirror=https://www.apache.org/dyn/closer.lua/flink/flink-1.17.2
-Dflink.archive.name=flink-1.17.2-bin-scala_2.12.tgz'
comment: 'verify-on-flink-1.17-binary'
- java: 8
flink: '1.20'
- flink-archive:
'-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.18.1
-Dflink.archive.name=flink-1.18.1-bin-scala_2.12.tgz'
+ flink-archive:
'-Dflink.archive.mirror=https://www.apache.org/dyn/closer.lua/flink/flink-1.18.1
-Dflink.archive.name=flink-1.18.1-bin-scala_2.12.tgz'
comment: 'verify-on-flink-1.18-binary'
- java: 8
flink: '1.20'
- flink-archive:
'-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.19.1
-Dflink.archive.name=flink-1.19.1-bin-scala_2.12.tgz'
+ flink-archive:
'-Dflink.archive.mirror=https://www.apache.org/dyn/closer.lua/flink/flink-1.19.1
-Dflink.archive.name=flink-1.19.1-bin-scala_2.12.tgz'
comment: 'verify-on-flink-1.19-binary'
steps:
- uses: actions/checkout@v4
@@ -317,10 +317,10 @@ jobs:
comment: [ "normal" ]
include:
- java: 8
- hive-archive:
'-Dhive.archive.mirror=https://archive.apache.org/dist/hive/hive-2.3.10
-Dhive.archive.name=apache-hive-2.3.10-bin.tar.gz'
+ hive-archive:
'-Dhive.archive.mirror=https://www.apache.org/dyn/closer.lua/hive/hive-2.3.10
-Dhive.archive.name=apache-hive-2.3.10-bin.tar.gz'
comment: 'verify-on-hive-2.3-binary'
- java: 8
- hive-archive:
'-Dhive.archive.mirror=https://github.com/pan3793/cdh-hive/releases/download/cdh6.3.2-release
-Dhive.archive.name=apache-hive-2.1.1-cdh6.3.2-bin.tar.gz'
+ hive-archive:
'-Dhive.archive.mirror=https://github.com/pan3793/cdh-hive/releases/download/cdh6.3.2-release
-Dhive.archive.name=apache-hive-2.1.1-cdh6.3.2-bin.tar.gz
-Dhive.archive.query='
comment: 'verify-on-hive-2.1-cdh6-binary'
steps:
- uses: actions/checkout@v4
diff --git a/docs/contributing/code/building.md
b/docs/contributing/code/building.md
index bc507bd3fb..689b705f7b 100644
--- a/docs/contributing/code/building.md
+++ b/docs/contributing/code/building.md
@@ -82,7 +82,7 @@ For the Scala version for Spark engines, the server will look
up the `SPARK_SCAL
## Building With Apache dlcdn Site
-By default, we use `https://archive.apache.org/dist/` to download the built-in
release packages of engines,
+By default, we use
[`closer.lua`](https://infra.apache.org/release-download-pages.html#download-scripts)
to download the built-in release packages of engines,
such as Spark or Flink.
But sometimes, you may find it hard to reach, or the download speed is too
slow,
then you can define the `apache.archive.dist` by `-Pmirror-cdn` to accelerate
to download speed.
diff --git a/externals/kyuubi-download/pom.xml
b/externals/kyuubi-download/pom.xml
index cd72e8e5d0..a7554dba63 100644
--- a/externals/kyuubi-download/pom.xml
+++ b/externals/kyuubi-download/pom.xml
@@ -51,7 +51,8 @@
<phase>compile</phase>
<configuration>
<skip>${spark.archive.download.skip}</skip>
-
<url>${spark.archive.mirror}/${spark.archive.name}</url>
+
<url>${spark.archive.mirror}/${spark.archive.name}${spark.archive.query}</url>
+
<outputFileName>${spark.archive.name}</outputFileName>
</configuration>
</execution>
<execution>
@@ -62,7 +63,8 @@
<phase>compile</phase>
<configuration>
<skip>${flink.archive.download.skip}</skip>
-
<url>${flink.archive.mirror}/${flink.archive.name}</url>
+
<url>${flink.archive.mirror}/${flink.archive.name}${flink.archive.query}</url>
+
<outputFileName>${flink.archive.name}</outputFileName>
</configuration>
</execution>
<execution>
@@ -73,7 +75,8 @@
<phase>compile</phase>
<configuration>
<skip>${hive.archive.download.skip}</skip>
-
<url>${hive.archive.mirror}/${hive.archive.name}</url>
+
<url>${hive.archive.mirror}/${hive.archive.name}${hive.archive.query}</url>
+
<outputFileName>${hive.archive.name}</outputFileName>
</configuration>
</execution>
</executions>
diff --git a/pom.xml b/pom.xml
index 2a63d7da7c..25d1e212db 100644
--- a/pom.xml
+++ b/pom.xml
@@ -125,7 +125,7 @@
<!-- Please don't upgrade the version to 4.10+, it depends on JDK 11
-->
<antlr4.version>4.9.3</antlr4.version>
<antlr.st4.version>4.3.4</antlr.st4.version>
-
<apache.archive.dist>https://archive.apache.org/dist</apache.archive.dist>
+
<apache.archive.dist>https://www.apache.org/dyn/closer.lua</apache.archive.dist>
<atlas.version>2.3.0</atlas.version>
<byte-buddy.version>1.14.15</byte-buddy.version>
<bouncycastle.version>1.78</bouncycastle.version>
@@ -143,6 +143,7 @@
<flink.version>1.20.0</flink.version>
<flink.archive.name>flink-${flink.version}-bin-scala_2.12.tgz</flink.archive.name>
<flink.archive.mirror>${apache.archive.dist}/flink/flink-${flink.version}</flink.archive.mirror>
+ <flink.archive.query>?action=download</flink.archive.query>
<flink.archive.download.skip>false</flink.archive.download.skip>
<google.jsr305.version>3.0.2</google.jsr305.version>
<grpc.version>1.65.1</grpc.version>
@@ -154,6 +155,7 @@
<hive.version>3.1.3</hive.version>
<hive.archive.name>apache-hive-${hive.version}-bin.tar.gz</hive.archive.name>
<hive.archive.mirror>${apache.archive.dist}/hive/hive-${hive.version}</hive.archive.mirror>
+ <hive.archive.query>?action=download</hive.archive.query>
<hive.archive.download.skip>false</hive.archive.download.skip>
<httpclient.version>4.5.14</httpclient.version>
<httpcore.version>4.4.16</httpcore.version>
@@ -204,6 +206,7 @@
<spark.archive.scala.suffix></spark.archive.scala.suffix>
<spark.archive.name>spark-${spark.version}-bin-hadoop3${spark.archive.scala.suffix}.tgz</spark.archive.name>
<spark.archive.mirror>${apache.archive.dist}/spark/spark-${spark.version}</spark.archive.mirror>
+ <spark.archive.query>?action=download</spark.archive.query>
<spark.archive.download.skip>false</spark.archive.download.skip>
<sqlite.version>3.46.1.3</sqlite.version>
<supercsv.version>2.2.0</supercsv.version>
@@ -1905,6 +1908,9 @@
<properties>
<!-- the apache cdn mirror works only for latest apache
releases -->
<apache.archive.dist>https://dlcdn.apache.org</apache.archive.dist>
+ <spark.archive.query></spark.archive.query>
+ <flink.archive.query></flink.archive.query>
+ <hive.archive.query></hive.archive.query>
<nodeDownloadRoot>https://npmmirror.com/mirrors/node/</nodeDownloadRoot>
<pnpmDownloadRoot>https://registry.npmmirror.com/pnpm/-/</pnpmDownloadRoot>
</properties>
