This is an automated email from the ASF dual-hosted git repository.
guyuqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 9d575858a BIGTOP-4016. Upgrade Spark to 3.3.3. (#1190)
9d575858a is described below
commit 9d575858a4b6d2afc5cb1de23ca2c8dc18e78c59
Author: Kengo Seki <[email protected]>
AuthorDate: Wed Oct 18 11:49:31 2023 +0900
BIGTOP-4016. Upgrade Spark to 3.3.3. (#1190)
---
bigtop-packages/src/common/spark/LICENSE-binary | 30 ++++++----------------
.../src/common/spark/do-component-build | 4 ++-
.../common/spark/licenses-binary/LICENSE-blas.txt | 25 ++++++++++++++++++
.../spark/licenses-binary/LICENSE-bootstrap.txt | 29 +++++++++++++--------
.../spark/licenses-binary/LICENSE-datatables.txt | 4 +--
bigtop.bom | 2 +-
6 files changed, 58 insertions(+), 36 deletions(-)
diff --git a/bigtop-packages/src/common/spark/LICENSE-binary
b/bigtop-packages/src/common/spark/LICENSE-binary
index b50da6be4..40e2e389b 100644
--- a/bigtop-packages/src/common/spark/LICENSE-binary
+++ b/bigtop-packages/src/common/spark/LICENSE-binary
@@ -218,7 +218,6 @@ javax.jdo:jdo-api
joda-time:joda-time
net.sf.opencsv:opencsv
org.apache.derby:derby
-org.ehcache:ehcache
org.objenesis:objenesis
org.roaringbitmap:RoaringBitmap
org.scalanlp:breeze-macros_2.12
@@ -261,7 +260,6 @@ net.sf.supercsv:super-csv
org.apache.arrow:arrow-format
org.apache.arrow:arrow-memory
org.apache.arrow:arrow-vector
-org.apache.commons:commons-configuration2
org.apache.commons:commons-crypto
org.apache.commons:commons-lang3
org.apache.hadoop:hadoop-annotations
@@ -296,7 +294,6 @@ org.apache.kerby:kerby-config
org.apache.kerby:kerby-pkix
org.apache.kerby:kerby-util
org.apache.kerby:kerby-xdr
-org.apache.kerby:token-provider
org.apache.orc:orc-core
org.apache.orc:orc-mapreduce
org.mortbay.jetty:jetty
@@ -316,19 +313,15 @@ com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider
com.fasterxml.jackson.module:jackson-module-jaxb-annotations
com.fasterxml.jackson.module:jackson-module-paranamer
com.fasterxml.jackson.module:jackson-module-scala_2.12
-com.fasterxml.woodstox:woodstox-core
com.github.mifmif:generex
-com.github.stephenc.jcip:jcip-annotations
com.google.code.findbugs:jsr305
com.google.code.gson:gson
com.google.flatbuffers:flatbuffers-java
com.google.guava:guava
com.google.inject:guice
com.google.inject.extensions:guice-servlet
-com.nimbusds:nimbus-jose-jwt
com.twitter:parquet-hadoop-bundle
commons-cli:commons-cli
-commons-daemon:commons-daemon
commons-dbcp:commons-dbcp
commons-io:commons-io
commons-lang:commons-lang
@@ -340,8 +333,6 @@ javax.inject:javax.inject
javax.validation:validation-api
log4j:apache-log4j-extras
log4j:log4j
-net.minidev:accessors-smart
-net.minidev:json-smart
net.sf.jpam:jpam
org.apache.avro:avro
org.apache.avro:avro-ipc
@@ -357,7 +348,6 @@ org.apache.directory.server:apacheds-i18n
org.apache.directory.server:apacheds-kerberos-codec
org.apache.htrace:htrace-core
org.apache.ivy:ivy
-org.apache.geronimo.specs:geronimo-jcache_1.0_spec
org.apache.mesos:mesos
org.apache.parquet:parquet-column
org.apache.parquet:parquet-common
@@ -432,7 +422,6 @@ BSD 2-Clause
------------
com.github.luben:zstd-jni
-dnsjava:dnsjava
javolution:javolution
com.esotericsoftware:kryo-shaded
com.esotericsoftware:minlog
@@ -440,7 +429,6 @@ com.esotericsoftware:reflectasm
com.google.protobuf:protobuf-java
org.codehaus.janino:commons-compiler
org.codehaus.janino:janino
-org.codehaus.woodstox:stax2-api
jline:jline
org.jodd:jodd-core
com.github.wendykierp:JTransforms
@@ -456,8 +444,6 @@ org.antlr:ST4
org.antlr:stringtemplate
org.antlr:antlr4-runtime
antlr:antlr
-com.github.fommil.netlib:core
-com.google.re2j:re2j
com.thoughtworks.paranamer:paranamer
org.scala-lang:scala-compiler
org.scala-lang:scala-library
@@ -470,6 +456,7 @@ net.sf.py4j:py4j
org.jpmml:pmml-model
org.jpmml:pmml-schema
org.threeten:threeten-extra
+org.jdom:jdom2
python/lib/py4j-*-src.zip
python/pyspark/cloudpickle.py
@@ -492,12 +479,15 @@ org.typelevel:spire-util_2.12
org.typelevel:algebra_2.12:jar
org.typelevel:cats-kernel_2.12
org.typelevel:machinist_2.12
-net.razorvine:pyrolite
+net.razorvine:pickle
org.slf4j:jcl-over-slf4j
org.slf4j:jul-to-slf4j
org.slf4j:slf4j-api
org.slf4j:slf4j-log4j12
com.github.scopt:scopt_2.12
+dev.ludovic.netlib:blas
+dev.ludovic.netlib:arpack
+dev.ludovic.netlib:lapack
core/src/main/resources/org/apache/spark/ui/static/dagre-d3.min.js
core/src/main/resources/org/apache/spark/ui/static/*dataTables*
@@ -515,13 +505,13 @@ Common Development and Distribution License (CDDL) 1.0
javax.activation:activation
http://www.oracle.com/technetwork/java/javase/tech/index-jsp-138795.html
javax.xml.stream:stax-api https://jcp.org/en/jsr/detail?id=173
javax.transaction:javax.transaction-api
+javax.xml.bind:jaxb-api
Common Development and Distribution License (CDDL) 1.1
------------------------------------------------------
javax.el:javax.el-api https://javaee.github.io/uel-ri/
-javax.servlet:javax.servlet-api https://javaee.github.io/servlet-spec/
javax.servlet.jsp:jsp-api
javax.transaction:jta http://www.oracle.com/technetwork/java/index.html
javax.xml.bind:jaxb-api https://github.com/javaee/jaxb-v2
@@ -553,14 +543,10 @@ Eclipse Public License (EPL) 2.0
--------------------------------
jakarta.annotation:jakarta-annotation-api
https://projects.eclipse.org/projects/ee4j.ca
+jakarta.servlet:jakarta.servlet-api
https://projects.eclipse.org/projects/ee4j.servlet
jakarta.ws.rs:jakarta.ws.rs-api https://github.com/eclipse-ee4j/jaxrs-api
org.glassfish.hk2.external:jakarta.inject
-
-
-Python Software Foundation License
-----------------------------------
-
-pyspark/heapq3.py
+com.github.jnr:jnr-posix
Public Domain
diff --git a/bigtop-packages/src/common/spark/do-component-build
b/bigtop-packages/src/common/spark/do-component-build
index 54e5b8591..14a5cc745 100644
--- a/bigtop-packages/src/common/spark/do-component-build
+++ b/bigtop-packages/src/common/spark/do-component-build
@@ -22,8 +22,10 @@ BUILD_OPTS="-Divy.home=${HOME}/.ivy2
-Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${
-Drepo.maven.org=$IVY_MIRROR_PROP \
-Dreactor.repo=file://${HOME}/.m2/repository \
-Dhadoop.version=$HADOOP_VERSION \
+ -Dkafka.version=$KAFKA_VERSION \
+ -Dzookeeper.version=$ZOOKEEPER_VERSION \
-Dyarn.version=$HADOOP_VERSION \
- -Pyarn -Phadoop-3.2 \
+ -Pyarn -Phadoop-3 \
-Phive -Phive-thriftserver \
-Psparkr -Pkubernetes \
-Pscala-${SCALA_VERSION%.*} \
diff --git a/bigtop-packages/src/common/spark/licenses-binary/LICENSE-blas.txt
b/bigtop-packages/src/common/spark/licenses-binary/LICENSE-blas.txt
new file mode 100644
index 000000000..2b8bec28b
--- /dev/null
+++ b/bigtop-packages/src/common/spark/licenses-binary/LICENSE-blas.txt
@@ -0,0 +1,25 @@
+MIT License
+-----------
+
+Copyright 2020, 2021, Ludovic Henry
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+Please contact [email protected] or visit ludovic.dev if you need additional
+information or have any questions.
diff --git
a/bigtop-packages/src/common/spark/licenses-binary/LICENSE-bootstrap.txt
b/bigtop-packages/src/common/spark/licenses-binary/LICENSE-bootstrap.txt
index 6c711832f..7d1e2311f 100644
--- a/bigtop-packages/src/common/spark/licenses-binary/LICENSE-bootstrap.txt
+++ b/bigtop-packages/src/common/spark/licenses-binary/LICENSE-bootstrap.txt
@@ -1,13 +1,22 @@
-Copyright 2013 Twitter, Inc.
+The MIT License (MIT)
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+Copyright (c) 2011-2019 Twitter, Inc.
+Copyright (c) 2011-2019 The Bootstrap Authors
-http://www.apache.org/licenses/LICENSE-2.0
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git
a/bigtop-packages/src/common/spark/licenses-binary/LICENSE-datatables.txt
b/bigtop-packages/src/common/spark/licenses-binary/LICENSE-datatables.txt
index bb7708b5b..d1fac3bfb 100644
--- a/bigtop-packages/src/common/spark/licenses-binary/LICENSE-datatables.txt
+++ b/bigtop-packages/src/common/spark/licenses-binary/LICENSE-datatables.txt
@@ -1,7 +1,7 @@
-Copyright (C) 2008-2018, SpryMedia Ltd.
+Copyright (C) 2008-2020, SpryMedia Ltd.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
\ No newline at end of file
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
diff --git a/bigtop.bom b/bigtop.bom
index e6dbe455a..c6d91914c 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -223,7 +223,7 @@ bigtop {
* when upgrading spark version.
* See comments in [bigtop-packages/src/common/spark/install_spark.sh]
for details.
*/
- version { base = '3.2.3'; pkg = base; release = 1 }
+ version { base = '3.3.3'; pkg = base; release = 1 }
tarball { destination = "$name-${version.base}.tar.gz"
source = "$name-${version.base}.tgz" }
url { download_path = "/$name/$name-${version.base}"
