Ottomata has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/387663 )
Change subject: Initial debian release (2.1.2-bin-hadoop2.6-1) ...................................................................... Initial debian release (2.1.2-bin-hadoop2.6-1) Change-Id: I33d7d1329499d3bfd47fb6a2b3944b3b7b879f9b --- A debian/README.Debian A debian/bin/pyspark2 A debian/bin/spark2-beeline A debian/bin/spark2-shell A debian/bin/spark2-sql A debian/bin/spark2-submit A debian/bin/spark2R A debian/changelog A debian/compat A debian/conf/log4j.properties A debian/conf/spark-defaults.conf A debian/conf/spark-env.sh A debian/control A debian/copyright A debian/gbp.conf A debian/rules A debian/source/format A debian/source/include-binaries A debian/source/options A debian/spark2.dirs A debian/spark2.install A debian/spark2.links A debian/spark2.postinst A debian/spark2.postrm 24 files changed, 566 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/debs/spark2 refs/changes/63/387663/1 diff --git a/debian/README.Debian b/debian/README.Debian new file mode 100644 index 0000000..e929737 --- /dev/null +++ b/debian/README.Debian @@ -0,0 +1,23 @@ +Druid for Debian +---------------- + +This package is created from the release tarballs provided from +https://spark.apache.org/downloads.html. It does not build Spark from source. + +This repository was created using git-import-orig: + + VERSION=2.1.2-bin-hadoop2.6 + gbp import-orig -u $VERSION --upstream-branch=master --debian-branch=debian ../spark-$VERSION.tgz + +You should be able to import a new upstream release using the same command again. + +Once you've imported the Spark 2 tarball, you need to explicitly declare all +binary files that need to be included in the package. The following command should do the trick. + + find {jars,python/lib,yarn,R} -type f -exec file {} \; | grep -v text | awk -F ':' '{print $1}' | sort > debian/source/include-binaries + +Finally, to build --include-removal needs to be passed to dpkg-source so that any removed binaries +will be removed from the source package. You can do so by passing --source-option="--include-removal" +to git-buildpackage or dpkg-buildpackage. On a Wikimedia build server: + + GIT_PBUILDER_AUTOCONF=no DIST=jessie WIKIMEDIA=yes gbp buildpackage -sa -us -uc --git-builder=git-pbuilder --source-option="--include-removal" diff --git a/debian/bin/pyspark2 b/debian/bin/pyspark2 new file mode 100755 index 0000000..a051128 --- /dev/null +++ b/debian/bin/pyspark2 @@ -0,0 +1 @@ +exec /usr/lib/spark2/bin/pyspark "$@" diff --git a/debian/bin/spark2-beeline b/debian/bin/spark2-beeline new file mode 100755 index 0000000..d94a75b --- /dev/null +++ b/debian/bin/spark2-beeline @@ -0,0 +1 @@ +exec /usr/lib/spark2/bin/beeline "$@" diff --git a/debian/bin/spark2-shell b/debian/bin/spark2-shell new file mode 100755 index 0000000..1b93163 --- /dev/null +++ b/debian/bin/spark2-shell @@ -0,0 +1 @@ +exec /usr/lib/spark2/bin/spark-shell "$@" diff --git a/debian/bin/spark2-sql b/debian/bin/spark2-sql new file mode 100755 index 0000000..9a1b180 --- /dev/null +++ b/debian/bin/spark2-sql @@ -0,0 +1 @@ +exec /usr/lib/spark2/bin/spark-sql "$@" diff --git a/debian/bin/spark2-submit b/debian/bin/spark2-submit new file mode 100755 index 0000000..1b6952c --- /dev/null +++ b/debian/bin/spark2-submit @@ -0,0 +1 @@ +exec /usr/lib/spark2/bin/spark-submit "$@" diff --git a/debian/bin/spark2R b/debian/bin/spark2R new file mode 100755 index 0000000..318d730 --- /dev/null +++ b/debian/bin/spark2R @@ -0,0 +1 @@ +exec /usr/lib/spark2/bin/sparkR "$@" diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..7697342 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +spark2 (2.1.2-bin-hadoop2.6-1) jessie-wikimedia; urgency=low + + * 2.1.2 binary release for Hadoop 2.6 + + -- Andrew Otto (WMF) <[email protected]> Tue, 31 Oct 2017 18:04:29 +0000 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +9 diff --git a/debian/conf/log4j.properties b/debian/conf/log4j.properties new file mode 100644 index 0000000..ec1aa18 --- /dev/null +++ b/debian/conf/log4j.properties @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the console +log4j.rootCategory=INFO, console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n + +# Set the default spark-shell log level to WARN. When running the spark-shell, the +# log level for this class is used to overwrite the root logger's log level, so that +# the user can have different defaults for the shell and regular Spark apps. +log4j.logger.org.apache.spark.repl.Main=WARN + +# Settings to quiet third party logs that are too verbose +log4j.logger.org.spark_project.jetty=WARN +log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR +log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO +log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO +log4j.logger.org.apache.parquet=ERROR +log4j.logger.parquet=ERROR + +# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support +log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL +log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR diff --git a/debian/conf/spark-defaults.conf b/debian/conf/spark-defaults.conf new file mode 100644 index 0000000..32368ba --- /dev/null +++ b/debian/conf/spark-defaults.conf @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Default system properties included when running spark-submit. +# This is useful for setting default environmental settings. + +# Example: +# spark.master spark://master:7077 +# spark.eventLog.enabled true +# spark.eventLog.dir hdfs://namenode:8021/directory +# spark.serializer org.apache.spark.serializer.KryoSerializer +# spark.driver.memory 5g +# spark.executor.extraJavaOptions -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three" + +# Dynamic allocation allows Spark to dynamically scale the cluster resources +# allocated for an application based on the workload. Only available in YARN mode. +# More info: https://spark.apache.org/docs/2.1.2/configuration.html#dynamic-allocation +spark.dynamicAllocation.enabled true +spark.shuffle.service.enabled true +spark.dynamicAllocation.executorIdleTimeout 60s +spark.dynamicAllocation.cachedExecutorIdleTimeout 3600s diff --git a/debian/conf/spark-env.sh b/debian/conf/spark-env.sh new file mode 100755 index 0000000..4d5a14c --- /dev/null +++ b/debian/conf/spark-env.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This file is sourced when running various Spark programs. +# Copy it as spark-env.sh and edit that to configure Spark for your site. + +# Options read when launching programs locally with +# ./bin/run-example or ./bin/spark-submit +# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files +# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node +# - SPARK_PUBLIC_DNS, to set the public dns name of the driver program +# - SPARK_CLASSPATH, default classpath entries to append + +# Options read by executors and drivers running inside the cluster +# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node +# - SPARK_PUBLIC_DNS, to set the public DNS name of the driver program +# - SPARK_CLASSPATH, default classpath entries to append +# - SPARK_LOCAL_DIRS, storage directories to use on this node for shuffle and RDD data +# - MESOS_NATIVE_JAVA_LIBRARY, to point to your libmesos.so if you use Mesos + +# Options read in YARN client mode +# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files +# - SPARK_EXECUTOR_INSTANCES, Number of executors to start (Default: 2) +# - SPARK_EXECUTOR_CORES, Number of cores for the executors (Default: 1). +# - SPARK_EXECUTOR_MEMORY, Memory per Executor (e.g. 1000M, 2G) (Default: 1G) +# - SPARK_DRIVER_MEMORY, Memory for Driver (e.g. 1000M, 2G) (Default: 1G) + +# Options for the daemons used in the standalone deploy mode +# - SPARK_MASTER_HOST, to bind the master to a different IP address or hostname +# - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports for the master +# - SPARK_MASTER_OPTS, to set config properties only for the master (e.g. "-Dx=y") +# - SPARK_WORKER_CORES, to set the number of cores to use on this machine +# - SPARK_WORKER_MEMORY, to set how much total memory workers have to give executors (e.g. 1000m, 2g) +# - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT, to use non-default ports for the worker +# - SPARK_WORKER_INSTANCES, to set the number of worker processes per node +# - SPARK_WORKER_DIR, to set the working directory of worker processes +# - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. "-Dx=y") +# - SPARK_DAEMON_MEMORY, to allocate to the master, worker and history server themselves (default: 1g). +# - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y") +# - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y") +# - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y") +# - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers + +# Generic options for the daemons used in the standalone deploy mode +# - SPARK_CONF_DIR Alternate conf dir. (Default: ${SPARK_HOME}/conf) +# - SPARK_LOG_DIR Where log files are stored. (Default: ${SPARK_HOME}/logs) +# - SPARK_PID_DIR Where the pid file is stored. (Default: /tmp) +# - SPARK_IDENT_STRING A string representing this instance of spark. (Default: $USER) +# - SPARK_NICENESS The scheduling priority for daemons. (Default: 0) +# - SPARK_NO_DAEMONIZE Run the proposed command in the foreground. It will not output a PID file. + +# If /etc/hadoop/conf exists, use it as HADOOP_CONF_DIR +if [ -z "${HADOOP_CONF_DIR}" -a -e "/etc/hadoop/conf" ]; then + export HADOOP_CONF_DIR=/etc/hadoop/conf +fi diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..3daae1c --- /dev/null +++ b/debian/control @@ -0,0 +1,17 @@ +Source: spark2 +Section: misc +Priority: optional +Maintainer: Andrew Otto <[email protected]> +Build-Depends: debhelper (>= 9) +Standards-Version: 3.9.3 +Homepage: https://spark.apache.org + +Package: spark2 +Architecture: all +Depends: default-jre-headless | java7-runtime-headless, ${misc:Depends} +Description: Spark 2 + This is a binary distribution of Spark 2. This package can be installed + alongside of an existing Spark 1 installation. Spark executables are renamed + to have 'spark2' in the name. This package only installs libary files and + executable scripts; it does not include init scripts to run Spark daemons + in standalone mode. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..8dad7ba --- /dev/null +++ b/debian/copyright @@ -0,0 +1,50 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: Spark +Source: https://spark.apache.org + +Files: * +Copyright: Copyright 2014 and onwards The Apache Software Foundation. +License: Apache-2 + +Files: debian/* +Copyright: 2016 Andrew Otto <[email protected]> +License: Apache-2 + +License: Apache-2 + Licensed under the Apache License, Version 2.0 (the "License"); you may not + use this file except in compliance with the License. You may obtain a copy of + the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language overning permissions and limitations under + the License. + . + On Debian systems, the Apache 2.0 license can be found in + /usr/share/common-licenses/Apache-2.0. + +License: BSD-3-clause + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + . + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/debian/gbp.conf b/debian/gbp.conf new file mode 100644 index 0000000..fcebf3f --- /dev/null +++ b/debian/gbp.conf @@ -0,0 +1,7 @@ +[DEFAULT] +upstream-tree=tag +debian-branch=debian +debian-tag=debian/%(version)s +# this is how we invoke pbuilder, arguments passed to git-buildpackage will be +# passed to dpkg-buildpackge in the chroot +builder = /usr/bin/git-pbuilder diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..14110db --- /dev/null +++ b/debian/rules @@ -0,0 +1,32 @@ +#!/usr/bin/make -f + +%: + dh $@ + +override_dh_install: + dh_install + + # Copy libs into usr/lib/spark2/ + cp -rv $(CURDIR)/jars/* $(CURDIR)/debian/spark2/usr/lib/spark2/jars/ + cp -rv $(CURDIR)/R/* $(CURDIR)/debian/spark2/usr/lib/spark2/R/ + cp -rv $(CURDIR)/yarn/* $(CURDIR)/debian/spark2/usr/lib/spark2/yarn/ + # (Avoid copying python test files by specifying the files we want from python/ directory.) + cp -rv $(CURDIR)/python/docs $(CURDIR)/debian/spark2/usr/lib/spark2/python/ + cp -rv $(CURDIR)/python/lib $(CURDIR)/debian/spark2/usr/lib/spark2/python/ + cp -rv $(CURDIR)/python/pyspark $(CURDIR)/debian/spark2/usr/lib/spark2/python/ + cp -rv $(CURDIR)/python/MANIFEST.in $(CURDIR)/debian/spark2/usr/lib/spark2/python/ + cp -rv $(CURDIR)/python/pylintrc $(CURDIR)/debian/spark2/usr/lib/spark2/python/ + cp -rv $(CURDIR)/python/README.md $(CURDIR)/debian/spark2/usr/lib/spark2/python/ + cp -rv $(CURDIR)/python/setup.cfg $(CURDIR)/debian/spark2/usr/lib/spark2/python/ + cp -rv $(CURDIR)/python/setup.py $(CURDIR)/debian/spark2/usr/lib/spark2/python/ + + # Copy desired bin scripts + cp -v $(CURDIR)/bin/beeline $(CURDIR)/debian/spark2/usr/lib/spark2/bin/beeline + cp -v $(CURDIR)/bin/find-spark-home $(CURDIR)/debian/spark2/usr/lib/spark2/bin/find-spark-home + cp -v $(CURDIR)/bin/load-spark-env.sh $(CURDIR)/debian/spark2/usr/lib/spark2/bin/load-spark-env.sh + cp -v $(CURDIR)/bin/pyspark $(CURDIR)/debian/spark2/usr/lib/spark2/bin/pyspark + cp -v $(CURDIR)/bin/spark-class $(CURDIR)/debian/spark2/usr/lib/spark2/bin/spark-class + cp -v $(CURDIR)/bin/spark-shell $(CURDIR)/debian/spark2/usr/lib/spark2/bin/spark-shell + cp -v $(CURDIR)/bin/spark-sql $(CURDIR)/debian/spark2/usr/lib/spark2/bin/spark-sql + cp -v $(CURDIR)/bin/spark-submit $(CURDIR)/debian/spark2/usr/lib/spark2/bin/spark-submit + cp -v $(CURDIR)/bin/sparkR $(CURDIR)/debian/spark2/usr/lib/spark2/bin/sparkR diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/debian/source/include-binaries b/debian/source/include-binaries new file mode 100644 index 0000000..527a4b2 --- /dev/null +++ b/debian/source/include-binaries @@ -0,0 +1,223 @@ +R/lib/SparkR/Meta/Rd.rds +R/lib/SparkR/Meta/features.rds +R/lib/SparkR/Meta/hsearch.rds +R/lib/SparkR/Meta/links.rds +R/lib/SparkR/Meta/nsInfo.rds +R/lib/SparkR/Meta/package.rds +R/lib/SparkR/Meta/vignette.rds +R/lib/SparkR/R/SparkR.rdb +R/lib/SparkR/R/SparkR.rdx +R/lib/SparkR/help/SparkR.rdb +R/lib/SparkR/help/SparkR.rdx +R/lib/SparkR/help/aliases.rds +R/lib/SparkR/help/paths.rds +R/lib/sparkr.zip +jars/JavaEWAH-0.3.2.jar +jars/RoaringBitmap-0.5.11.jar +jars/ST4-4.0.4.jar +jars/activation-1.1.1.jar +jars/antlr-2.7.7.jar +jars/antlr-runtime-3.4.jar +jars/antlr4-runtime-4.5.3.jar +jars/aopalliance-1.0.jar +jars/aopalliance-repackaged-2.4.0-b34.jar +jars/apache-log4j-extras-1.2.17.jar +jars/apacheds-i18n-2.0.0-M15.jar +jars/apacheds-kerberos-codec-2.0.0-M15.jar +jars/api-asn1-api-1.0.0-M20.jar +jars/api-util-1.0.0-M20.jar +jars/arpack_combined_all-0.1.jar +jars/avro-1.7.7.jar +jars/avro-ipc-1.7.7.jar +jars/avro-mapred-1.7.7-hadoop2.jar +jars/base64-2.3.8.jar +jars/bcprov-jdk15on-1.51.jar +jars/bonecp-0.8.0.RELEASE.jar +jars/breeze-macros_2.11-0.12.jar +jars/breeze_2.11-0.12.jar +jars/calcite-avatica-1.2.0-incubating.jar +jars/calcite-core-1.2.0-incubating.jar +jars/calcite-linq4j-1.2.0-incubating.jar +jars/chill-java-0.8.0.jar +jars/chill_2.11-0.8.0.jar +jars/commons-beanutils-1.7.0.jar +jars/commons-beanutils-core-1.8.0.jar +jars/commons-cli-1.2.jar +jars/commons-codec-1.10.jar +jars/commons-collections-3.2.2.jar +jars/commons-compiler-3.0.0.jar +jars/commons-compress-1.4.1.jar +jars/commons-configuration-1.6.jar +jars/commons-crypto-1.0.0.jar +jars/commons-dbcp-1.4.jar +jars/commons-digester-1.8.jar +jars/commons-httpclient-3.1.jar +jars/commons-io-2.4.jar +jars/commons-lang-2.6.jar +jars/commons-lang3-3.5.jar +jars/commons-logging-1.1.3.jar +jars/commons-math3-3.4.1.jar +jars/commons-net-2.2.jar +jars/commons-pool-1.5.4.jar +jars/compress-lzf-1.0.3.jar +jars/core-1.1.2.jar +jars/curator-client-2.6.0.jar +jars/curator-framework-2.6.0.jar +jars/curator-recipes-2.6.0.jar +jars/datanucleus-api-jdo-3.2.6.jar +jars/datanucleus-core-3.2.10.jar +jars/datanucleus-rdbms-3.2.9.jar +jars/derby-10.12.1.1.jar +jars/eigenbase-properties-1.1.5.jar +jars/gson-2.2.4.jar +jars/guava-14.0.1.jar +jars/guice-3.0.jar +jars/guice-servlet-3.0.jar +jars/hadoop-annotations-2.6.4.jar +jars/hadoop-auth-2.6.4.jar +jars/hadoop-client-2.6.4.jar +jars/hadoop-common-2.6.4.jar +jars/hadoop-hdfs-2.6.4.jar +jars/hadoop-mapreduce-client-app-2.6.4.jar +jars/hadoop-mapreduce-client-common-2.6.4.jar +jars/hadoop-mapreduce-client-core-2.6.4.jar +jars/hadoop-mapreduce-client-jobclient-2.6.4.jar +jars/hadoop-mapreduce-client-shuffle-2.6.4.jar +jars/hadoop-yarn-api-2.6.4.jar +jars/hadoop-yarn-client-2.6.4.jar +jars/hadoop-yarn-common-2.6.4.jar +jars/hadoop-yarn-server-common-2.6.4.jar +jars/hadoop-yarn-server-web-proxy-2.6.4.jar +jars/hive-beeline-1.2.1.spark2.jar +jars/hive-cli-1.2.1.spark2.jar +jars/hive-exec-1.2.1.spark2.jar +jars/hive-jdbc-1.2.1.spark2.jar +jars/hive-metastore-1.2.1.spark2.jar +jars/hk2-api-2.4.0-b34.jar +jars/hk2-locator-2.4.0-b34.jar +jars/hk2-utils-2.4.0-b34.jar +jars/htrace-core-3.0.4.jar +jars/httpclient-4.5.2.jar +jars/httpcore-4.4.4.jar +jars/ivy-2.4.0.jar +jars/jackson-annotations-2.6.5.jar +jars/jackson-core-2.6.5.jar +jars/jackson-core-asl-1.9.13.jar +jars/jackson-databind-2.6.5.jar +jars/jackson-jaxrs-1.9.13.jar +jars/jackson-mapper-asl-1.9.13.jar +jars/jackson-module-paranamer-2.6.5.jar +jars/jackson-module-scala_2.11-2.6.5.jar +jars/jackson-xc-1.9.13.jar +jars/janino-3.0.0.jar +jars/java-xmlbuilder-1.0.jar +jars/javassist-3.18.1-GA.jar +jars/javax.annotation-api-1.2.jar +jars/javax.inject-1.jar +jars/javax.inject-2.4.0-b34.jar +jars/javax.servlet-api-3.1.0.jar +jars/javax.ws.rs-api-2.0.1.jar +jars/javolution-5.5.1.jar +jars/jaxb-api-2.2.2.jar +jars/jcl-over-slf4j-1.7.16.jar +jars/jdo-api-3.0.1.jar +jars/jersey-client-2.22.2.jar +jars/jersey-common-2.22.2.jar +jars/jersey-container-servlet-2.22.2.jar +jars/jersey-container-servlet-core-2.22.2.jar +jars/jersey-guava-2.22.2.jar +jars/jersey-media-jaxb-2.22.2.jar +jars/jersey-server-2.22.2.jar +jars/jets3t-0.9.3.jar +jars/jetty-6.1.26.jar +jars/jetty-util-6.1.26.jar +jars/jline-2.12.1.jar +jars/joda-time-2.9.3.jar +jars/jodd-core-3.5.2.jar +jars/jpam-1.1.jar +jars/json4s-ast_2.11-3.2.11.jar +jars/json4s-core_2.11-3.2.11.jar +jars/json4s-jackson_2.11-3.2.11.jar +jars/jsr305-1.3.9.jar +jars/jta-1.1.jar +jars/jtransforms-2.4.0.jar +jars/jul-to-slf4j-1.7.16.jar +jars/kryo-shaded-3.0.3.jar +jars/leveldbjni-all-1.8.jar +jars/libfb303-0.9.3.jar +jars/libthrift-0.9.3.jar +jars/log4j-1.2.17.jar +jars/lz4-1.3.0.jar +jars/mail-1.4.7.jar +jars/mesos-1.0.0-shaded-protobuf.jar +jars/metrics-core-3.1.2.jar +jars/metrics-graphite-3.1.2.jar +jars/metrics-json-3.1.2.jar +jars/metrics-jvm-3.1.2.jar +jars/minlog-1.3.0.jar +jars/mx4j-3.0.2.jar +jars/netty-3.8.0.Final.jar +jars/netty-all-4.0.43.Final.jar +jars/objenesis-2.1.jar +jars/opencsv-2.3.jar +jars/oro-2.0.8.jar +jars/osgi-resource-locator-1.0.1.jar +jars/paranamer-2.3.jar +jars/parquet-column-1.8.1.jar +jars/parquet-common-1.8.1.jar +jars/parquet-encoding-1.8.1.jar +jars/parquet-format-2.3.0-incubating.jar +jars/parquet-hadoop-1.8.1.jar +jars/parquet-hadoop-bundle-1.6.0.jar +jars/parquet-jackson-1.8.1.jar +jars/pmml-model-1.2.15.jar +jars/pmml-schema-1.2.15.jar +jars/protobuf-java-2.5.0.jar +jars/py4j-0.10.4.jar +jars/pyrolite-4.13.jar +jars/scala-compiler-2.11.8.jar +jars/scala-library-2.11.8.jar +jars/scala-parser-combinators_2.11-1.0.4.jar +jars/scala-reflect-2.11.8.jar +jars/scala-xml_2.11-1.0.2.jar +jars/scalap-2.11.8.jar +jars/shapeless_2.11-2.0.0.jar +jars/slf4j-api-1.7.16.jar +jars/slf4j-log4j12-1.7.16.jar +jars/snappy-0.2.jar +jars/snappy-java-1.1.2.6.jar +jars/spark-catalyst_2.11-2.1.2.jar +jars/spark-core_2.11-2.1.2.jar +jars/spark-graphx_2.11-2.1.2.jar +jars/spark-hive-thriftserver_2.11-2.1.2.jar +jars/spark-hive_2.11-2.1.2.jar +jars/spark-launcher_2.11-2.1.2.jar +jars/spark-mesos_2.11-2.1.2.jar +jars/spark-mllib-local_2.11-2.1.2.jar +jars/spark-mllib_2.11-2.1.2.jar +jars/spark-network-common_2.11-2.1.2.jar +jars/spark-network-shuffle_2.11-2.1.2.jar +jars/spark-repl_2.11-2.1.2.jar +jars/spark-sketch_2.11-2.1.2.jar +jars/spark-sql_2.11-2.1.2.jar +jars/spark-streaming_2.11-2.1.2.jar +jars/spark-tags_2.11-2.1.2.jar +jars/spark-unsafe_2.11-2.1.2.jar +jars/spark-yarn_2.11-2.1.2.jar +jars/spire-macros_2.11-0.7.4.jar +jars/spire_2.11-0.7.4.jar +jars/stax-api-1.0-2.jar +jars/stax-api-1.0.1.jar +jars/stream-2.7.0.jar +jars/stringtemplate-3.2.1.jar +jars/super-csv-2.2.0.jar +jars/univocity-parsers-2.2.1.jar +jars/validation-api-1.1.0.Final.jar +jars/xbean-asm5-shaded-4.4.jar +jars/xercesImpl-2.9.1.jar +jars/xmlenc-0.52.jar +jars/xz-1.0.jar +jars/zookeeper-3.4.6.jar +python/lib/py4j-0.10.4-src.zip +python/lib/pyspark.zip +yarn/spark-2.1.2-yarn-shuffle.jar diff --git a/debian/source/options b/debian/source/options new file mode 100644 index 0000000..2df3e3f --- /dev/null +++ b/debian/source/options @@ -0,0 +1 @@ +extend-diff-ignore = '^\.gitreview$' diff --git a/debian/spark2.dirs b/debian/spark2.dirs new file mode 100644 index 0000000..05542ed --- /dev/null +++ b/debian/spark2.dirs @@ -0,0 +1,9 @@ +etc/spark2 +etc/spark2/conf +usr/bin +usr/lib/spark2 +usr/lib/spark2/bin +usr/lib/spark2/jars +usr/lib/spark2/python +usr/lib/spark2/R +usr/lib/spark2/yarn diff --git a/debian/spark2.install b/debian/spark2.install new file mode 100644 index 0000000..1e9e414 --- /dev/null +++ b/debian/spark2.install @@ -0,0 +1,9 @@ +debian/bin/spark2-beeline usr/bin +debian/bin/pyspark2 usr/bin +debian/bin/spark2-shell usr/bin +debian/bin/spark2-sql usr/bin +debian/bin/spark2-submit usr/bin +debian/bin/spark2R usr/bin +debian/conf/log4j.properties etc/spark2/conf +debian/conf/spark-defaults.conf etc/spark2/conf +debian/conf/spark-env.sh etc/spark2/conf diff --git a/debian/spark2.links b/debian/spark2.links new file mode 100644 index 0000000..a0486bb --- /dev/null +++ b/debian/spark2.links @@ -0,0 +1 @@ +etc/spark2/conf usr/lib/spark2/conf diff --git a/debian/spark2.postinst b/debian/spark2.postinst new file mode 100644 index 0000000..c965bf3 --- /dev/null +++ b/debian/spark2.postinst @@ -0,0 +1,18 @@ +#!/bin/sh + +# postinst script for spark2 +# +# see: dh_installdeb(1) + +set -e + +# Symlink hive-site.xml into spark2/conf if it exists. +# This lets spark2 infer Hive configuration. +test -f /etc/hive/conf/hive-site.xml && ln -sf /etc/hive/conf/hive-site.xml /etc/spark2/conf/hive-site.xml + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/debian/spark2.postrm b/debian/spark2.postrm new file mode 100644 index 0000000..fc9f5c4 --- /dev/null +++ b/debian/spark2.postrm @@ -0,0 +1,17 @@ +#!/bin/sh + +# postrm script for spark2 +# +# see: dh_installdeb(1) + +set -e + +# Remove the hive-site.xml symlink if it exists +test -e /etc/hive/usr/lib/spark2/conf/hive-site.xml && unlink /etc/hive/usr/lib/spark2/conf/hive-site.xml + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 -- To view, visit https://gerrit.wikimedia.org/r/387663 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I33d7d1329499d3bfd47fb6a2b3944b3b7b879f9b Gerrit-PatchSet: 1 Gerrit-Project: operations/debs/spark2 Gerrit-Branch: debian Gerrit-Owner: Ottomata <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
