lanking520 closed pull request #12481: [MXNET-722][WIP][DO NOT MERGE] Maven
publish pipeline demo
URL: https://github.com/apache/incubator-mxnet/pull/12481
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/Jenkinsfile b/Jenkinsfile
index 346cb19ce46..a8f3d8ba36d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -654,6 +654,18 @@ core_logic: {
}
}
},
+ 'Scala: CPU Maven Publish': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/ut-scala-cpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git()
+ utils.unpack_lib('cpu', mx_dist_lib)
+ utils.docker_run('ubuntu_cpu',
'unittest_ubuntu_cpu_scala_publish', false)
+ utils.publish_test_coverage()
+ }
+ }
+ }
+ },
'Scala: CPU': {
node(NODE_LINUX_CPU) {
ws('workspace/ut-scala-cpu') {
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 0b6a42c3cda..f7dc2cc7135 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -268,7 +268,6 @@ build_amzn_linux_cpu() {
ninja -v
}
-
build_centos7_mkldnn() {
set -ex
cd /work/mxnet
@@ -713,6 +712,12 @@ unittest_ubuntu_python3_quantization_gpu() {
nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file
nosetests_quantization_gpu.xml --verbose tests/python/quantization_gpu
}
+unittest_ubuntu_cpu_scala_publish() {
+ set -ex
+ cd /work/mxnet
+ sudo bash scala-package/dev/build.sh
+}
+
unittest_ubuntu_cpu_scala() {
set -ex
make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1
diff --git a/scala-package/dev/build.sh b/scala-package/dev/build.sh
new file mode 100644
index 00000000000..9c618efcf70
--- /dev/null
+++ b/scala-package/dev/build.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Install Dependencies
+# sudo bash ci/docker/install/ubuntu_core.sh
+# sudo bash ci/docker/install/ubuntu_scala.sh
+
+# Setup Environment Variables
+# OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu
+export OS_TYPE=linux-x86_64-cpu
+
+# This script is used to build the base dependencies of MXNet Scala Env
+# git clone --recursive https://github.com/apache/incubator-mxnet
+# cd incubator-mxnet/
+git checkout origin/master -b maven
+apt-get install libssl-dev
+
+# This part is used to build the gpg dependencies:
+apt-get install -y maven gnupg gnupg2 gnupg-agent
+mv /usr/bin/gpg /usr/bin/gpg1
+ln -s /usr/bin/gpg2 /usr/bin/gpg
+
+# Run python to configure keys
+python3 $PWD/scala-package/dev/buildkey.py
+
+# Updating cache
+mkdir -p ~/.gnupg
+echo "default-cache-ttl 14400" > ~/.gnupg/gpg-agent.conf
+echo "max-cache-ttl 14400" >> ~/.gnupg/gpg-agent.conf
+export GPG_TTY=$(tty)
+
+# Build the Scala MXNet backend
+bash scala-package/dev/compile-mxnet-backend.sh $OS_TYPE ./
+# Scala steps to deploy
+make scalapkg
+make scalaunittest
+make scalaintegrationtest
+make scalarelease-dryrun
+make scaladeploy
diff --git a/scala-package/dev/buildkey.py b/scala-package/dev/buildkey.py
new file mode 100644
index 00000000000..e345fc2f0ea
--- /dev/null
+++ b/scala-package/dev/buildkey.py
@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import subprocess
+import time
+
+'''
+Thhis file would do the following items:
+ Import keys from AWS Credential services
+ Create settings.xml in .m2 with pass phrase
+ Create security-settings.xml in .m2 with master password
+ Import keys.asc the encrypted keys in gpg
+'''
+
+def importASC(path, passPhrase):
+ subprocess.run(['gpg', '--batch', '--yes',
+ '--passphrase=\"{}\"'.format(passPhrase),
+ "--import", "{}/keys.asc".format(path)])
+
+def encryptMasterPSW(password):
+ result = subprocess.run(['mvn', '--encrypt-master-password', password],
+ stdout=subprocess.PIPE)
+ return str(result.stdout)[2:-3]
+
+def encryptPSW(password):
+ result = subprocess.run(['mvn', '--encrypt-password', password],
+ stdout=subprocess.PIPE)
+ return str(result.stdout)[2:-3]
+
+def createASC(path, data):
+ f = open(path + "keys.asc", "w+")
+ f.write(data)
+ f.close()
+
+def masterPSW(path, password):
+ f = open(path + "settings-security.xml", "w+")
+ f.write("<settingsSecurity>\n <master>{}</master>\n</settingsSecurity>"
+ .format(password))
+ f.close()
+
+def severPSW(path, username, password, passPhrase):
+ f = open(path + "settings.xml", "w+")
+ f.write(
+ '''
+ <?xml version="1.0" encoding="UTF-8"?>
+ <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
http://maven.apache.org/xsd/settings-1.0.0.xsd";>
+ <pluginGroups></pluginGroups>
+ <proxies></proxies>
+ <servers>
+ <server>
+ <id>apache.snapshots.https</id>
+ <username>{}</username>
+ <password>{}</password>
+ </server>
+ <!-- To stage a release of some part of Maven -->
+ <server>
+ <id>apache.releases.https</id>
+ <username>{}</username>
+ <password>{}</password>
+ </server>
+ </servers>
+ <mirrors></mirrors>
+ <profiles>
+ <profile>
+ <id>gpg</id>
+ <properties>
+ <gpg.executable>gpg2</gpg.executable>
+ <gpg.passphrase>{}</gpg.passphrase>
+ </properties>
+ </profile>
+ </profiles>
+ <activeProfiles>
+ <activeProfile>gpg</activeProfile>
+ </activeProfiles>
+ </settings>
+ '''.format(username, password, username, password, passPhrase))
+
+if __name__ == "__main__":
+ # Assume I got the secret key from secret services
+ # secret = get_secret()
+ userCredential = {"username": "nswamy", "password": "123456"}
+ keyCredential = {
+ "passPhrase" : "asdfg", "encrytedKey": "A-Very-long-string",
+ "masterPass": "123456"}
+ masterPass = encryptMasterPSW(keyCredential["masterPass"])
+ masterPSW("~/.m2/", masterPass)
+ passwordEncrypted = encryptPSW(userCredential["password"])
+ severPSW("~/.m2/", userCredential["username"], passwordEncrypted,
+ keyCredential["passPhrase"])
+ # createASC("./", keyCredential["encrytedKey"])
+ # importASC(keyCredential["passPhrase"])
+
diff --git a/scala-package/dev/get_credential.py
b/scala-package/dev/get_credential.py
new file mode 100644
index 00000000000..e64c9cc1110
--- /dev/null
+++ b/scala-package/dev/get_credential.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import boto3
+from botocore.exceptions import ClientError
+
+
+def get_secret():
+ secret_name = "credentials/<credentials>"
+ endpoint_url = "https://secretsmanager.us-west-2.amazonaws.com";
+ region_name = "us-west-2"
+
+ session = boto3.session.Session()
+ client = session.client(
+ service_name='secretsmanager',
+ region_name=region_name,
+ endpoint_url=endpoint_url
+ )
+
+ try:
+ get_secret_value_response = client.get_secret_value(
+ SecretId=secret_name
+ )
+ except ClientError as e:
+ if e.response['Error']['Code'] == 'ResourceNotFoundException':
+ print("The requested secret " + secret_name + " was not found")
+ elif e.response['Error']['Code'] == 'InvalidRequestException':
+ print("The request was invalid due to:", e)
+ elif e.response['Error']['Code'] == 'InvalidParameterException':
+ print("The request had invalid params:", e)
+ else:
+ # Decrypted secret using the associated KMS CMK
+ # Depending on whether the secret was a string or binary, one of these
fields will be populated
+ if 'SecretString' in get_secret_value_response:
+ secret = get_secret_value_response['SecretString']
+ else:
+ binary_secret_data = get_secret_value_response['SecretBinary']
+ return secret, binary_secret_data
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
