This is an automated email from the ASF dual-hosted git repository.
guyuqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new d590ea671 BIGTOP-3893: Fix Hadoop3.3.4 build issues on ppc64le (#1077)
d590ea671 is described below
commit d590ea671237297514864dbe1c8f32baa26255bd
Author: Yuqi Gu <[email protected]>
AuthorDate: Mon Dec 19 10:08:16 2022 +0800
BIGTOP-3893: Fix Hadoop3.3.4 build issues on ppc64le (#1077)
---
.../src/common/hadoop/do-component-build | 7 +++++
bigtop_toolchain/manifests/grpc.pp | 30 +++++++++++++++++++---
bigtop_toolchain/manifests/protobuf.pp | 14 ++++++++++
3 files changed, 47 insertions(+), 4 deletions(-)
diff --git a/bigtop-packages/src/common/hadoop/do-component-build
b/bigtop-packages/src/common/hadoop/do-component-build
index 27703d862..60ff0de96 100644
--- a/bigtop-packages/src/common/hadoop/do-component-build
+++ b/bigtop-packages/src/common/hadoop/do-component-build
@@ -39,6 +39,13 @@ if [ $HOSTTYPE = "powerpc64le" ] ; then
#cleanup
rm -rf ${LEVELDBJNI_HOME}
rm -rf ${LEVELDB_HOME}
+
+ mvn install:install-file -DgroupId=com.google.protobuf
-DartifactId=protoc -Dversion=2.5.0 \
+ -Dclassifier=linux-ppcle_64 -Dpackaging=exe
-Dfile=/usr/local/protobuf-2.5.0/bin/protoc
+ mvn install:install-file -DgroupId=com.google.protobuf
-DartifactId=protoc -Dversion=3.6.1 \
+ -Dclassifier=linux-ppcle_64 -Dpackaging=exe
-Dfile=/usr/local/protobuf-3.6.1/bin/protoc
+ mvn install:install-file -DgroupId=io.grpc
-DartifactId=protoc-gen-grpc-java -Dversion=1.26.0 \
+ -Dclassifier=linux-ppcle_64 -Dpackaging=exe
-Dfile=/usr/src/grpc-java-1.26.0/compiler/build/exe/java_plugin/protoc-gen-grpc-java
fi
## BIGTOP-2288
diff --git a/bigtop_toolchain/manifests/grpc.pp
b/bigtop_toolchain/manifests/grpc.pp
index 4c9e41088..acb0a41d7 100644
--- a/bigtop_toolchain/manifests/grpc.pp
+++ b/bigtop_toolchain/manifests/grpc.pp
@@ -13,35 +13,57 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# There is no pre-built binary of grpc-java for ppc64le before
grpc-java-1.48.0 (2022-08-01).
+# Build some versions of grpc-java Binaries in Bigtop toolchain.
class bigtop_toolchain::grpc {
require bigtop_toolchain::jdk
require bigtop_toolchain::protobuf
$grpc_version = '1.28.0'
- $proto_version = '3.17.3'
$proto_home = "/usr/local/protobuf-3.17.3"
if ($architecture == 'ppc64le') {
- exec { "download grpc-java ${grpc_version}":
+ # grpc-java 1.28.0 + protobuf-3.17.3 for zeppelin build on ppc64le.
+ exec { "download grpc-java ${grpc_version}":
cwd => "/usr/src",
command => "/usr/bin/wget
https://github.com/grpc/grpc-java/archive/refs/tags/v${grpc_version}.tar.gz &&
mkdir -p grpc-java-${grpc_version} && /bin/tar -xvzf v${grpc_version}.tar.gz -C
grpc-java-${grpc_version} --strip-components=1",
creates => "/usr/src/grpc-java-${grpc_version}",
- }
+ }
file {
"/usr/src/grpc-java-${grpc_version}/grpc-java-${grpc_version}-add-support-for-ppc64le.patch":
source =>
"puppet:///modules/bigtop_toolchain/grpc-java-${grpc_version}-add-support-for-ppc64le.patch",
require => Exec["download grpc-java ${grpc_version}"],
}
- exec { "build grpc-java ${grpc_version}":
+ exec { "build grpc-java ${grpc_version}":
cwd => "/usr/src/grpc-java-${grpc_version}",
command => "/usr/bin/patch -p1 <
grpc-java-${grpc_version}-add-support-for-ppc64le.patch && export
LDFLAGS=-L/${proto_home}/lib && export CXXFLAGS=-I/${proto_home}/include &&
export LD_LIBRARY_PATH=/${proto_home}/lib && cd compiler && ../gradlew
java_pluginExecutable -PskipAndroid=true",
creates => "/usr/local/grpc-java-${grpc_version}",
require =>
File["/usr/src/grpc-java-${grpc_version}/grpc-java-${grpc_version}-add-support-for-ppc64le.patch"],
timeout => 3000
}
+
+ # grpc-java 1.26.0 + protobuf-3.6.1 for Hadoop-3.3.4 build on ppc64le.
+ exec { "download grpc-java 1.26.0":
+ cwd => "/usr/src",
+ command => "/usr/bin/wget
https://github.com/grpc/grpc-java/archive/refs/tags/v1.26.0.tar.gz && mkdir -p
grpc-java-1.26.0 && /bin/tar -xvzf v1.26.0.tar.gz -C grpc-java-1.26.0
--strip-components=1",
+ creates => "/usr/src/grpc-java-1.26.0",
+ }
+
+ file {
"/usr/src/grpc-java-1.26.0/grpc-java-1.26.0-add-support-for-ppc64le.patch":
+ source =>
"puppet:///modules/bigtop_toolchain/grpc-java-${grpc_version}-add-support-for-ppc64le.patch",
+ require => Exec["download grpc-java ${grpc_version}"],
+ }
+
+ exec { "build grpc-java 1.26.0":
+ cwd => "/usr/src/grpc-java-1.26.0",
+ command => "/usr/bin/patch -p1 <
grpc-java-1.26.0-add-support-for-ppc64le.patch && export
LDFLAGS=-L//usr/local/protobuf-3.6.1/lib && export
CXXFLAGS=-I//usr/local/protobuf-3.6.1/include && export
LD_LIBRARY_PATH=//usr/local/protobuf-3.6.1/lib && cd compiler && ../gradlew
java_pluginExecutable -PskipAndroid=true",
+ creates => "/usr/local/grpc-java-1.26.0",
+ require =>
File["/usr/src/grpc-java-1.26.0/grpc-java-1.26.0-add-support-for-ppc64le.patch"],
+ timeout => 3000
+ }
}
}
diff --git a/bigtop_toolchain/manifests/protobuf.pp
b/bigtop_toolchain/manifests/protobuf.pp
index 120cbfab7..3dedd49a7 100644
--- a/bigtop_toolchain/manifests/protobuf.pp
+++ b/bigtop_toolchain/manifests/protobuf.pp
@@ -87,6 +87,20 @@ class bigtop_toolchain::protobuf {
require => Exec["download protobuf 3.17.3"],
timeout => 3000
}
+
+ exec { "download protobuf 3.6.1":
+ cwd => "/usr/src",
+ command => "/usr/bin/wget
https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.6.1.tar.gz &&
mkdir -p protobuf-3.6.1 && /bin/tar -xvzf v3.6.1.tar.gz -C protobuf-3.6.1
--strip-components=1",
+ creates => "/usr/src/protobuf-3.6.1",
+ }
+
+ exec { "install protobuf 3.6.1":
+ cwd => "/usr/src/protobuf-3.6.1",
+ command => "/usr/src/protobuf-3.6.1/autogen.sh &&
/usr/src/protobuf-3.6.1/configure --prefix=/usr/local/protobuf-3.6.1
--disable-shared --with-pic && /usr/bin/make install",
+ creates => "/usr/local/protobuf-3.6.1",
+ require => Exec["download protobuf 3.6.1"],
+ timeout => 3000
+ }
}
}
