This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new cfb8acc30 BIGTOP-4044: Enhance Bigtop with Concurrent Compilation
Support for Additional Components (#1212)
cfb8acc30 is described below
commit cfb8acc30f22b1a129dd657d6c308d74ea13bd1f
Author: jialiang <[email protected]>
AuthorDate: Thu Dec 28 14:39:23 2023 +0800
BIGTOP-4044: Enhance Bigtop with Concurrent Compilation Support for
Additional Components (#1212)
---
bigtop.bom | 9 +++++++++
packages.gradle | 38 ++++++++++++++++++++++++++++++++++++++
2 files changed, 47 insertions(+)
diff --git a/bigtop.bom b/bigtop.bom
index bce880831..02c0b433c 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -86,6 +86,7 @@
packaging = 'rpm' // *optional* If this component can be built only as
either
// DEB or RPM, specify that packaging format
explicitly.
// If both formats are supported, omit this option.
+ maven_parallel_build = true
}
}
}
@@ -146,6 +147,7 @@ bigtop {
site = "${apache.APACHE_MIRROR}/${download_path}"
archive = "${apache.APACHE_ARCHIVE}/${download_path}"
}
+ maven_parallel_build = true
}
'hadoop' {
name = 'hadoop'
@@ -168,6 +170,7 @@ bigtop {
url { download_path = "/$name/${version.base}/"
site = "${apache.APACHE_MIRROR}/${download_path}"
archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
+ maven_parallel_build = true
}
'hive' {
@@ -180,6 +183,7 @@ bigtop {
url { download_path = "/$name/$name-${version.base}/"
site = "${apache.APACHE_MIRROR}/${download_path}"
archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
+ maven_parallel_build = true
}
'tez' {
name = 'tez'
@@ -231,6 +235,7 @@ bigtop {
url { download_path = "/$name/$name-${version.base}"
site = "${apache.APACHE_MIRROR}/${download_path}"
archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
+ maven_parallel_build = true
}
@@ -243,6 +248,7 @@ bigtop {
url { download_path = "/$name/$name-${version.base}"
site = "${apache.APACHE_MIRROR}/${download_path}"
archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
+ maven_parallel_build = true
}
'bigtop-groovy' {
name = 'bigtop-groovy'
@@ -286,6 +292,7 @@ bigtop {
source = "v${version.base}.tar.gz" }
url { site = "https://github.com/Alluxio/alluxio/archive";
archive = site }
+ maven_parallel_build = true
}
'kafka' {
name = 'kafka'
@@ -308,6 +315,7 @@ bigtop {
url { download_path = "/$name/$name-${version.base}/"
site = "${apache.APACHE_MIRROR}/${download_path}"
archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
+ maven_parallel_build = true
}
'gpdb' {
name = 'gpdb'
@@ -327,6 +335,7 @@ bigtop {
url { download_path = "incubator/livy/${version.base}-incubating/"
site = "${apache.APACHE_MIRROR}/${download_path}"
archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
+ maven_parallel_build = true
}
'ranger' {
name = 'ranger'
diff --git a/packages.gradle b/packages.gradle
index cfaabfc1d..d78c47c77 100644
--- a/packages.gradle
+++ b/packages.gradle
@@ -76,6 +76,27 @@ def safeDelete = { fileName ->
def getDate() {
new Date().format('E, dd MMM yyyy HH:mm:ss Z')
}
+
+def generatePatch(String threadCount, String path) {
+ def patchContent = """
+ diff --git a/.mvn/maven.config b/.mvn/maven.config
+ new file mode 100644
+ index 00000000..e69de29b
+ diff --git a/.mvn/maven.config b/.mvn/maven.config
+ index e69de29b..0bea21c9 100644
+ --- a/.mvn/maven.config
+ +++ b/.mvn/maven.config
+ @@ -0,0 +1 @@
+ +-T${threadCount}
+ \\ No newline at end of file
+ """.stripIndent()
+
+ new File(path, "patch0-maven-parallel-build.diff").with {
+ write(patchContent)
+ println "Patch file created at: ${absolutePath}"
+ }
+}
+
/**
* To avoid breaking the compat with existing packages let's use the old style
names
*/
@@ -137,6 +158,7 @@ task "bom-json" (description: "List the components of the
stack in json format")
pkg: it.value.pkg,
relNotes: it.value.relNotes,
packaging: it.value.packaging,
+ maven_parallel_build: it.value.maven_parallel_build
],
tarball: [
destination: it.value.tarball.destination,
@@ -394,6 +416,8 @@ def genTasks = { target ->
delete ("$PKG_BUILD_DIR/deb")
def final DEB_BLD_DIR = "$PKG_BUILD_DIR/deb/$NAME-${PKG_VERSION}"
def final DEB_PKG_DIR =
"$PKG_BUILD_DIR/deb/$PKG_NAME-${PKG_VERSION}-${BIGTOP_BUILD_STAMP}"
+ def final ENABLE_MAVEN_PARALLEL_BUILD =
config.bigtop.components[target].maven_parallel_build
+ def final MAVEN_BUILD_THREADS = project.hasProperty('buildThreads') ?
project.property('buildThreads') : null
mkdir (DEB_BLD_DIR)
copy {
from SEED_TAR
@@ -428,6 +452,12 @@ def genTasks = { target ->
}
into "$DEB_BLD_DIR/debian"
}
+ if (MAVEN_BUILD_THREADS && ENABLE_MAVEN_PARALLEL_BUILD) {
+ if (!MAVEN_BUILD_THREADS.matches("\\d+C")) {
+ throw new GradleException("Invalid MAVEN_BUILD_THREADS parameter. It
must be a combination of numbers and 'C', such as '2C'.")
+ }
+ generatePatch(MAVEN_BUILD_THREADS, "$DEB_BLD_DIR/debian")
+ }
// Prepeare bom file with all the versions
def bomWriter = new File("$DEB_BLD_DIR/debian/bigtop.bom").newWriter()
bomVersions.each { bomWriter << "$it\n"}
@@ -575,6 +605,8 @@ def genTasks = { target ->
def final PKG_VERSION = config.bigtop.components[target].version.pkg
def final BASE_VERSION = config.bigtop.components[target].version.base
def final PKG_OUTPUT_DIR = config.bigtop.components[target].outputdir
+ def final ENABLE_MAVEN_PARALLEL_BUILD =
config.bigtop.components[target].maven_parallel_build
+ def final MAVEN_BUILD_THREADS = project.hasProperty('buildThreads') ?
project.property('buildThreads') : null
safeDelete ("$PKG_BUILD_DIR/rpm")
['INSTALL','SOURCES','BUILD','SRPMS','RPMS'].each { rpmdir ->
mkdir("$PKG_BUILD_DIR/rpm/$rpmdir")
@@ -609,6 +641,12 @@ def genTasks = { target ->
}
into "$PKG_BUILD_DIR/rpm/SOURCES"
}
+ if (MAVEN_BUILD_THREADS && ENABLE_MAVEN_PARALLEL_BUILD) {
+ if (!MAVEN_BUILD_THREADS.matches("\\d+C")) {
+ throw new GradleException("Invalid MAVEN_BUILD_THREADS parameter. It
must be a combination of numbers and 'C', such as '2C'.")
+ }
+ generatePatch(MAVEN_BUILD_THREADS, "$PKG_BUILD_DIR/rpm/SOURCES")
+ }
// Writing bigtop.bom files with all the versions
def bomWriter = new
File("$PKG_BUILD_DIR/rpm/SOURCES/bigtop.bom").newWriter()
bomVersions.each { bomWriter << "$it\n"}
