Enrico Deusebio has proposed merging soss/+source/charmed-spark:dpe-1966-build-from-source-new into soss/+source/charmed-spark:lp-3.5.0.
Requested reviews: Mehdi B. (medib) Canonical Data Platform (data-platform) For more details, see: https://code.launchpad.net/~data-platform/soss/+source/charmed-spark/+git/charmed-spark/+merge/455623 First release of Spark 3.5.0-ubuntu0 -- Your team Canonical Data Platform is requested to review the proposed merge of soss/+source/charmed-spark:dpe-1966-build-from-source-new into soss/+source/charmed-spark:lp-3.5.0.
diff --git a/.launchpad.yaml b/.launchpad.yaml new file mode 100644 index 0000000..ba0eced --- /dev/null +++ b/.launchpad.yaml @@ -0,0 +1,109 @@ +pipeline: +- build + +jobs: + build: + series: jammy + architectures: amd64 + packages: + - wget + - openjdk-8-jdk + - maven + - git + - python3 + - python3-pip + - python3-setuptools + - zip + snaps: + - name: yq + classic: true + environment: + JAVA_OPTS: "-Dhttp.proxyHost=10.10.10.1 -Dhttp.proxyPort=8222 -Dhttps.proxyHost=10.10.10.1 -Dhttps.proxyPort=8222 -Dhttp.nonProxyHosts" + NO_PROXY: localhost + ARTIFACTORY_BUILDING_URL: "https://canonical.jfrog.io/artifactory/dataplatform-spark/"; + ARTIFACTORY_STAGING_URL: "https://canonical.jfrog.io/artifactory/dataplatform-spark-staging/"; + RELEASE_NAME: "k8s" + run: |- + # try to read branch name (works only locally) + + # We need to copy files under home due to the confined nature of the snap + cp pom.xml ~/pom.xml + SPARK_VERSION=$(yq .project.version ~/pom.xml) + rm ~/pom.xml + echo "Spark version: $SPARK_VERSION" + + BRANCH_NAME=$(git branch --show-current) + echo "branch_name: $BRANCH_NAME" + # check if branch name is valid + if [ -z "$BRANCH_NAME" ] + then + # get branch revision id from git HEAD file + echo "No branch name given from git command! Try to get it from .git folder" + git_rev=$(cat .git/HEAD) + while read line; do + current_rev=$( echo $line | awk -F ' ' '{print $1}' ) + branch_name=$( echo $line | awk -F ' ' '{print $2}' | awk -F '/' '{print $NF}' ) + if [[ $current_rev = $git_rev ]] + then + echo "Branch name: $branch_name" + export BRANCH_NAME=$branch_name + fi + done < .git/packed-refs + else + export JAVA_OPTS="" + sed -i 's/<proxies>/<!-- <proxies>/' settings.xml + sed -i 's/<\/proxies>/<\/proxies> -->/' settings.xml + fi + [ ! -z "$BRANCH_NAME" ] && echo "Current branch: $BRANCH_NAME" + if [[ "$BRANCH_NAME" != "lp-"* ]]; then + export ARTIFACTORY_URL=$ARTIFACTORY_STAGING_URL + export RELEASE=false + else + export ARTIFACTORY_URL=$ARTIFACTORY_BUILDING_URL + export RELEASE=true + fi + echo "Selected artifactory: $ARTIFACTORY_URL" + echo "Release artifact: $RELEASE" + # check artifactory credentials + [ -z "PIP_INDEX_URL" ] && exit 1 + [ ! -z "$PIP_INDEX_URL" ] && echo "Env variable exists :) " + [ ! -z "$PIP_INDEX_URL" ] && export USERNAME=$(echo "${PIP_INDEX_URL#https://}"; | awk -F '@' '{print $1}' | awk -F ':' '{print $1}') + [ ! -z "$PIP_INDEX_URL" ] && export PASSWORD=$(echo "${PIP_INDEX_URL#https://}"; | awk -F '@' '{print $1}' | awk -F ':' '{print $2}') + echo "ARTIFACTORY TO BE USED: $ARTIFACTORY_URL" + echo "ARTIFACTORY USERNAME: $USERNAME" + # check release name + current_date=$(date '+%Y%m%d%H%M%S') + CANONICAL_PATCH_VERSION=$(cat PATCH_VERSION ) + CANONICAL_PATCH_VERSION+="-$current_date" + export CANONICAL_PATCH_VERSION=$CANONICAL_PATCH_VERSION + echo "Canonical patch: $RELEASE_NAME" + # set java version to Java 8 + update-java-alternatives -s $(update-java-alternatives -l | grep '1\.8' | cut -d " " -f1) || echo '.' + java -version + # configure setting for maven repository + mkdir ~/.m2 + mv settings.xml ~/.m2/settings.xml + cat ~/.m2/settings.xml + # start building process + echo "Start building Spark..." + # build spark + ./dev/make-distribution.sh --pip --tgz --name $RELEASE_NAME -Pkubernetes -Phive -Phive-thriftserver + FILE=spark-$SPARK_VERSION-$CANONICAL_PATCH_VERSION-bin-$RELEASE_NAME.tgz + if [ -f "$FILE" ]; then + echo "$FILE exists. Spark is correctly built." + else + echo "$FILE does not exist. Exit..." + exit 1 + fi + # compute checksum + sha512sum spark-$SPARK_VERSION-$CANONICAL_PATCH_VERSION-bin-$RELEASE_NAME.tgz > spark-$SPARK_VERSION-$CANONICAL_PATCH_VERSION-bin-$RELEASE_NAME.tgz.sha512 + cd ~/.m2 + # compress local maven repository + zip -rq repository.zip repository + cp repository.zip /build/lpci/project/ + cd /build/lpci/project/ + output: + paths: + - spark-*-bin-*.tgz + - spark-*-bin-*.tgz.sha512 + - repository.zip diff --git a/PATCH_VERSION b/PATCH_VERSION new file mode 100644 index 0000000..852dc9e --- /dev/null +++ b/PATCH_VERSION @@ -0,0 +1 @@ +ubuntu0 diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh index ef7c010..3fa53ee 100755 --- a/dev/make-distribution.sh +++ b/dev/make-distribution.sh @@ -36,7 +36,7 @@ MAKE_TGZ=false MAKE_PIP=false MAKE_R=false NAME=none -MVN="$SPARK_HOME/build/mvn" +MVN="mvn" function exit_with_usage { set +x @@ -153,7 +153,7 @@ fi echo "Spark version is $VERSION" if [ "$MAKE_TGZ" == "true" ]; then - echo "Making spark-$VERSION-bin-$NAME.tgz" + echo "Making spark-$VERSION-$CANONICAL_PATCH_VERSION-bin-$NAME.tgz" else echo "Making distribution for Spark $VERSION in '$DISTDIR'..." fi @@ -289,7 +289,7 @@ if [ -d "$SPARK_HOME/R/lib/SparkR" ]; then fi if [ "$MAKE_TGZ" == "true" ]; then - TARDIR_NAME=spark-$VERSION-bin-$NAME + TARDIR_NAME=spark-$VERSION-$CANONICAL_PATCH_VERSION-bin-$NAME TARDIR="$SPARK_HOME/$TARDIR_NAME" rm -rf "$TARDIR" cp -r "$DISTDIR" "$TARDIR" @@ -297,6 +297,6 @@ if [ "$MAKE_TGZ" == "true" ]; then if [ "$(uname -s)" = "Darwin" ]; then TAR="tar --no-mac-metadata --no-xattrs --no-fflags" fi - $TAR -czf "spark-$VERSION-bin-$NAME.tgz" -C "$SPARK_HOME" "$TARDIR_NAME" + $TAR -czf "spark-$VERSION-$CANONICAL_PATCH_VERSION-bin-$NAME.tgz" -C "$SPARK_HOME" "$TARDIR_NAME" rm -rf "$TARDIR" fi diff --git a/pom.xml b/pom.xml index 93d696d..245d38a 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,7 @@ <java.version>1.8</java.version> <maven.compiler.source>${java.version}</maven.compiler.source> <maven.compiler.target>${java.version}</maven.compiler.target> - <maven.version>3.8.8</maven.version> + <maven.version>3.6.3</maven.version> <exec-maven-plugin.version>3.1.0</exec-maven-plugin.version> <sbt.project.name>spark</sbt.project.name> <asm.version>9.5</asm.version> @@ -175,7 +175,7 @@ <scala.version>2.12.18</scala.version> <scala.binary.version>2.12</scala.binary.version> <scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version> - <!-- dont update scala-maven-plugin to version 4.8.1 SPARK-42809 and SPARK-43595 --> + <!-- dont update scala-maven-plugin to version 4.8.1 SPARK-42809 and SPARK-43595 --> <scala-maven-plugin.version>4.8.0</scala-maven-plugin.version> <maven.scaladoc.skip>false</maven.scaladoc.skip> <versions-maven-plugin.version>2.16.0</versions-maven-plugin.version> diff --git a/settings.xml b/settings.xml new file mode 100644 index 0000000..dcb5cf7 --- /dev/null +++ b/settings.xml @@ -0,0 +1,81 @@ +<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd";> + <servers> + <server> + <username>${env.USERNAME}</username> + <password>${env.PASSWORD}</password> + <id>central</id> + </server> + <server> + <username>${env.USERNAME}</username> + <password>${env.PASSWORD}</password> + <id>snapshots</id> + </server> + </servers> + <profiles> + <profile> + <repositories> + <repository> + <snapshots> + <enabled>false</enabled> + </snapshots> + <id>central</id> + <name>dataplatform-spark</name> + <url>${env.ARTIFACTORY_URL}</url> + </repository> + <repository> + <snapshots /> + <id>snapshots</id> + <name>dataplatform-spark</name> + <url>${env.ARTIFACTORY_URL}</url> + </repository> + </repositories> + <pluginRepositories> + <pluginRepository> + <snapshots> + <enabled>false</enabled> + </snapshots> + <id>central</id> + <name>dataplatform-spark</name> + <url>${env.ARTIFACTORY_URL}</url> + </pluginRepository> + <pluginRepository> + <snapshots /> + <id>snapshots</id> + <name>dataplatform-spark</name> + <url>${env.ARTIFACTORY_URL}</url> + </pluginRepository> + </pluginRepositories> + <id>artifactory</id> + </profile> + </profiles> + <activeProfiles> + <activeProfile>artifactory</activeProfile> + </activeProfiles> + <proxies> + <proxy> + <id>http_proxy</id> + <active>true</active> + <protocol>http</protocol> + <host>10.10.10.1</host> + <port>8222</port> + <nonProxyHosts>localhost</nonProxyHosts> + </proxy> + <proxy> + <id>https_proxy</id> + <active>true</active> + <protocol>https</protocol> + <host>10.10.10.1</host> + <port>8222</port> + <nonProxyHosts>localhost</nonProxyHosts> + </proxy> + </proxies> + <mirrors> + <mirror> + <id>central</id> + <name>Maven Repository Manager running on canonical.jfrog.io</name> + <url>${env.ARTIFACTORY_URL}</url> + <mirrorOf>*</mirrorOf> + </mirror> + </mirrors> +</settings>
-- Mailing list: https://launchpad.net/~data-platform Post to : [email protected] Unsubscribe : https://launchpad.net/~data-platform More help : https://help.launchpad.net/ListHelp
