Ottomata has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/347606 )
Change subject: Move hadooop python package declaration into client
......................................................................
Move hadooop python package declaration into client
This will install packages on workers as well as clients (like stat100[24]).
Change-Id: Ib810b3247828f8400ac9bfb06a68c2a01a21c6f8
---
M modules/role/manifests/analytics_cluster/hadoop/client.pp
M modules/role/manifests/analytics_cluster/hadoop/worker.pp
2 files changed, 37 insertions(+), 39 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/06/347606/1
diff --git a/modules/role/manifests/analytics_cluster/hadoop/client.pp
b/modules/role/manifests/analytics_cluster/hadoop/client.pp
index c67b514..d6f65c5 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/client.pp
+++ b/modules/role/manifests/analytics_cluster/hadoop/client.pp
@@ -98,6 +98,43 @@
}
}
+ # Install packages that are useful for distributed
+ # computation in Hadoop, and thus should be available on
+ # any Hadoop nodes.
+ require_package(
+ 'python-pandas',
+ 'python-scipy',
+ 'python-requests',
+ 'python-matplotlib',
+ 'python-dateutil',
+ 'python-sympy',
+ 'python-docopt',
+ 'python3',
+ 'python3-tabulate',
+ 'python3-scipy',
+ 'python3-enchant',
+ 'python3-tz',
+ 'python3-nltk',
+ 'python3-nose',
+ 'python3-setuptools',
+ 'python3-requests',
+ 'python3-mmh3',
+ 'python3-docopt',
+ 'libgomp1'
+ )
+
+ # Need a specifc version of python-numpy for sklearn.
+ # There are some weird dependency / require_package
+ # issues that force us to use the package resource
+ # directly.
+ package { ['python-numpy', 'python3-numpy']:
+ ensure => '1:1.12.0-2~bpo8+1',
+ }
+ package { ['python3-sklearn','python3-sklearn-lib']:
+ ensure => 'installed',
+ require => Package['python3-numpy'],
+ }
+
if $::realm == 'labs' {
# Hadoop directories in labs should be created by puppet.
# This conditional could be added to each worker,master,standby
diff --git a/modules/role/manifests/analytics_cluster/hadoop/worker.pp
b/modules/role/manifests/analytics_cluster/hadoop/worker.pp
index d3fb532..731d24b 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/worker.pp
+++ b/modules/role/manifests/analytics_cluster/hadoop/worker.pp
@@ -114,45 +114,6 @@
# Install MaxMind databases for geocoding UDFs
include ::geoip
- # Install packages that are useful for distributed
- # computation in Hadoop, and thus should be available on
- # any Hadoop nodes.
- require_package(
- 'python-pandas',
- 'python-scipy',
- 'python-requests',
- 'python-matplotlib',
- 'python-dateutil',
- 'python-sympy',
- 'python-docopt',
- 'python3',
- 'python3-tabulate',
- 'python3-scipy',
- 'python3-enchant',
- 'python3-tz',
- 'python3-nltk',
- 'python3-nose',
- 'python3-setuptools',
- 'python3-requests',
- 'python3-mmh3',
- 'python3-docopt',
- 'libgomp1'
- )
-
- # Need a specifc version of python-numpy for sklearn.
- # There are some weird dependency / require_package
- # issues that force us to use the package resource
- # directly.
- package { ['python-numpy', 'python3-numpy']:
- ensure => '1:1.12.0-2~bpo8+1',
- }
- package { ['python3-sklearn','python3-sklearn-lib']:
- ensure => 'installed',
- require => Package['python3-numpy'],
- }
-
-
-
# This allows Hadoop daemons to talk to each other.
ferm::service{ 'hadoop-access':
proto => 'tcp',
--
To view, visit https://gerrit.wikimedia.org/r/347606
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib810b3247828f8400ac9bfb06a68c2a01a21c6f8
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
