Rush has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/345631 )
Change subject: nfs-mounts: per cluster definitions for mounts
......................................................................
nfs-mounts: per cluster definitions for mounts
* create cluster mappings in nfs-mounts.yaml
* move VOLUMES_NEEDING_EXPORTS logic into nfs-mounts.yaml
* get_cluster() can determine what cluster a host is a member of
* bind_path is dynamic from nfs-mounts for nfs-manage-binds
* nfs-mounts.yaml path is an argument for nfs-manage-binds
* nfs-manage-binds '-binds' happens now before public export binds
* move device mappings to nfs-mounts as per cluster
* separate project metadata (gid) into separate dict in nfs-mounts
* 'mounts' in nfs-mounts.yaml now has per cluster designations
* mount_nfs_volume now search through cluster dicts for first match
Bug: T158883
Change-Id: Icbc20ec86ea7057df2262350d8c8d1a052f8c8ad
---
M modules/labstore/files/nfs-exportd
M modules/labstore/files/nfs-manage-binds
M modules/labstore/files/nfs-mounts.yaml
M modules/labstore/lib/puppet/parser/functions/mount_nfs_volume.rb
4 files changed, 358 insertions(+), 245 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/31/345631/1
diff --git a/modules/labstore/files/nfs-exportd
b/modules/labstore/files/nfs-exportd
index 20f82a8..6f977e1 100755
--- a/modules/labstore/files/nfs-exportd
+++ b/modules/labstore/files/nfs-exportd
@@ -26,6 +26,7 @@
import time
import logging
import sys
+import socket
import subprocess
import keystoneauth1
@@ -33,9 +34,6 @@
from keystoneclient.auth.identity.v3 import Password as KeystonePassword
from novaclient import client as novaclient
-
-# Volumes that need an entry in exports.d
-VOLUMES_NEEDING_EXPORTS = ['project', 'home', 'tools-home', 'tools-project']
def is_valid_ipv4(ip):
@@ -110,26 +108,30 @@
return ips
-def get_projects_with_nfs(mounts_config, observer_pass):
- """
- Get populated project objects that need NFS exports
- :param mounts_config: dict
+def get_projects_with_nfs(cluster_projects,
+ cluster_mounts,
+ allowed_private,
+ observer_pass):
+ """ Get populated project objects that need NFS exports
+ :param cluster_projects: dict
+ :param cluster_mounts: dict
:returns: list
"""
projects = []
- for name, config in mounts_config['private'].items():
- if 'mounts' in config:
- mounts = [k for k, v in config['mounts'].items()
- if k in VOLUMES_NEEDING_EXPORTS and v]
- if len(mounts) == 0:
- # Skip project if it has no private mounts
- logging.debug('skipping exports for %s, no private mounts',
name)
- continue
- else:
+ for name, mount_list in cluster_mounts.items():
+
+ # do not generate per project exportd output for either public
+ # exports or defined private that are not white listed
+ mounts = [m for m in mount_list if m in allowed_private]
+
+ if len(mounts) == 0:
+ # Skip project if it has no private mounts
+ logging.debug('skipping exports for %s, no private mounts', name)
continue
+
ips = get_instance_ips(name, observer_pass)
if ips:
- project = Project(name, config['gid'], ips, mounts)
+ project = Project(name, cluster_projects[name], ips, mounts)
projects.append(project)
logging.debug('project %s has %s instances',
name, len(project.instance_ips))
@@ -175,12 +177,20 @@
return public_paths
-def write_project_exports(mounts_config, exports_d_path, observer_pass):
+def write_project_exports(cluster_projects,
+ cluster_mounts,
+ allowed_private,
+ exports_d_path,
+ observer_pass):
""" output project export definitions
:param mounts_config: dict of defined exports
"""
project_paths = []
- projects = get_projects_with_nfs(mounts_config, observer_pass)
+ projects = get_projects_with_nfs(cluster_projects,
+ cluster_mounts,
+ allowed_private,
+ observer_pass)
+
for project in projects:
logging.debug('writing exports file for %s', project.name)
path = os.path.join(exports_d_path, '%s.exports' % project.name)
@@ -188,6 +198,16 @@
f.write(project.get_exports())
project_paths.append(path)
return project_paths
+
+
+def get_cluster(clusters):
+ """:param clusters: dict
+ :returns: str
+ """
+ hostname = socket.gethostname()
+ for cluster, hosts in clusters.items():
+ if hostname in hosts:
+ return cluster
def main():
@@ -243,14 +263,21 @@
logging.exception('Could not load projects config file from %s',
args.config_path)
sys.exit(1)
+ cluster = get_cluster(config['clusters'])
+
exports_d_path = args.exports_d_path
existing_exports = [
os.path.join(exports_d_path, filename)
for filename in os.listdir(exports_d_path)]
- public_paths = write_public_exports(config['public'], exports_d_path)
- project_paths = write_project_exports(config, exports_d_path,
args.observer_pass)
+ public_paths = write_public_exports(config['public'][cluster],
exports_d_path)
+
+ project_paths = write_project_exports(config['projects'],
+ config['mounts'][cluster],
+ config['allowed_private_mounts'],
+ exports_d_path,
+ args.observer_pass)
# compile list of entries in export_d path that are not defined in
current config
existing_wo_public = list(set(existing_exports) - set(public_paths))
diff --git a/modules/labstore/files/nfs-manage-binds
b/modules/labstore/files/nfs-manage-binds
index 50d71ee..a5b37bb 100644
--- a/modules/labstore/files/nfs-manage-binds
+++ b/modules/labstore/files/nfs-manage-binds
@@ -3,6 +3,7 @@
import os
import logging
import sys
+import socket
import subprocess
import yaml
@@ -71,15 +72,16 @@
ensure_dir('%s/home' % target)
if 'project' in mounts:
ensure_dir('%s/project' % target)
+
if os.path.exists(target):
bind_mount(target, export)
else:
logging.warning("no bind on %s as %s does not exist" % (export,
target))
-def get_binds():
- """ find all bindmounts under /exp
- :note: we assume /exp is the root for exported binds
+def get_binds(bind_root):
+ """ find all bindmounts under bind_root
+ :param bind_root: str
:returns: list
"""
@@ -88,9 +90,18 @@
mnt_targets = subprocess.check_output(cmd).decode()
bind_mounts = []
for mnt in mnt_targets.split():
- if mnt.startswith('/exp'):
+ if mnt.startswith(bind_root):
bind_mounts.append(mnt.strip())
return bind_mounts
+
+def get_cluster(clusters):
+ """:param clusters: dict
+ :returns: str
+ """
+ hostname = socket.gethostname()
+ for cluster, hosts in clusters.items():
+ if hostname in hosts:
+ return cluster
def main():
@@ -100,7 +111,7 @@
argparser.add_argument(
'-f',
action='store_true',
- help='New project directories (not /exp directories) need -f to be
created.',
+ help='New project directories (not bind_path directories) need -f to
be created.',
)
argparser.add_argument(
@@ -110,14 +121,20 @@
)
argparser.add_argument(
- '-debug',
- help='Turn on debug logging',
+ '-binds',
+ help='Display active binds (this operation will exit post)',
action='store_true'
)
argparser.add_argument(
- '-binds',
- help='Display active binds (this operation will exit post)',
+ '-config_path',
+ default='/etc/nfs-mounts.yaml',
+ help='Path to YAML file containing config of which exports to
maintain',
+ )
+
+ argparser.add_argument(
+ '-debug',
+ help='Turn on debug logging',
action='store_true'
)
@@ -135,52 +152,87 @@
logging.warning("forcing creation for new project directories")
try:
- with open('/etc/nfs-mounts.yaml') as f:
+ with open(args.config_path) as f:
config = yaml.safe_load(f)
except:
logging.exception('Could not load projects config file from %s',
args.config_path)
sys.exit(1)
- if args.binds:
- binds = get_binds()
- if not binds:
- sys.exit(1)
- for b in binds:
- print(b)
- sys.exit(0)
+ cluster = get_cluster(config['clusters'])
+ if 'root' not in config['public'][cluster]:
+ logging.exception('No export root defined')
+ sys.exit(1)
+
+ public_binds = {}
srv_root = args.disk_path
# get dict of path_on_disk:share_name to create in descending order
- public_inverse = {v.split(' ')[0]: k for k, v in config['public'].items()}
+ public_inverse = {v.split(' ')[0]: k for k, v in
config['public'][cluster].items()}
for k in sorted(public_inverse, key=len, reverse=False):
path = k
if public_inverse[k] == 'root':
- exp_root = os.path.join('/', path)
+ exp_root = path
+ logging.info('Export root set to %s' % (exp_root,))
if not os.path.exists(exp_root):
logging.error('The root export path does not exist')
sys.exit(1)
continue
+ # process non-root public shares
base = os.path.basename(path)
exp = os.path.join(exp_root, base)
srv = os.path.join(srv_root, base)
- create_binding(srv, exp, force=args.f)
+ public_binds[srv] = exp
- device_path_default = '/srv/misc'
- device_paths = {
- 'tools': '/srv/tools',
- 'maps': '/srv/maps',
- }
+ if args.binds:
+ binds = get_binds(exp_root)
+ if not binds:
+ logging.warning("No binds found at %s", exp_root)
+ sys.exit(1)
+ for b in binds:
+ print(b)
+ sys.exit(0)
- for project in sorted(config['private']):
+ # set up non-project specific binds
+ for ppath, export in public_binds.items():
+ create_binding(ppath, export, force=args.f)
+
+ device_path_default = config['devices'][cluster]['default']
+ device_paths = config['devices'][cluster]['projects']
+
+ devices = list(device_paths.values()) + [device_path_default]
+ logging.debug("devices: %s" % (devices,))
+
+ for d in devices:
+ # Expect device paths to be mount points
+ if not is_mount(d):
+ logging.exception("%s is not a mount point" % d)
+ sys.exit(1)
+
+ # We do not export the root of these devices
+ # Expect a sysadmin has explicitly created a 'shared' dir
+ if not os.path.exists(os.path.join(d, 'shared')):
+ logging.exception("%s has no 'shared' directory" % d)
+ sys.exit(1)
+ else:
+ logging.debug("%s has a shared dir" % d)
+
+ for project in sorted(config['mounts'][cluster]):
+
# Find which mounts should be made available for project
# This is useful to create home and project dirs for new projects
- mount_config = config['private'][project].get('mounts', {})
- mounts = [mount for mount in mount_config.keys() if
mount_config[mount]]
-
+ mounts = config['mounts'][cluster][project]
srv_device = device_paths.get(project, device_path_default)
+
+ # If /dev/foo is mounted at /srv/foo then a 'shared' directory
+ # needs to be created /manually/ to seed the new device.
+
+ # Exported (via bind mount): /exp/myproject =>
/srv/foo/shared/myproject
+ # Example subtree (with subtree_check):
+ # * /srv/foo/shared/myproject/project
+ # * /srv/foo/shared/myproject/home
srv = os.path.join(srv_device, 'shared', project)
- exp = os.path.join('/exp/project', project)
+ exp = os.path.join(exp_root, 'project', project)
create_binding(srv, exp, force=args.f, mounts=mounts)
if __name__ == '__main__':
diff --git a/modules/labstore/files/nfs-mounts.yaml
b/modules/labstore/files/nfs-mounts.yaml
index ef88c2b..aee26fc 100644
--- a/modules/labstore/files/nfs-mounts.yaml
+++ b/modules/labstore/files/nfs-mounts.yaml
@@ -1,188 +1,216 @@
+# Clusters share storage state
+clusters:
+ primary: ['labstore1001', 'labstore1002']
+ secondary: ['labstore1004', 'labstore1005']
+ # For now server side is not managed
+ # dyanmically. Needs to be reimaged to Jessie.
+ tertiary: ['labstore1003']
+
+# Devices require a manual addition of a root
+# 'shared' directory to be viable for export on
+# per project resources
+devices:
+ secondary:
+ default: '/srv/misc'
+ projects:
+ tools: '/srv/tools'
+
+projects:
+ account-creation-assistance: 50088
+ bots: 50064
+ catgraph: 50588
+ tools: 50380
+ chasetestproject: 53308
+ contributors: 50408
+ cvn: 50116
+ wikidata-query: 52354
+ dumps: 50124
+ editor-engagement: 50068
+ etytree: 53231
+ fastcci: 50983
+ huggle: 50168
+ maps: 50196
+ maps-team: 52497
+ math: 50398
+ mwoffliner: 52376
+ openstack: 50076
+ osmit: 51933
+ project-proxy: 50254
+ quarry: 52243
+ snuggle: 50470
+ testlabs: 50302
+ tools: 50380
+ toolsbeta: 50610
+ twl: 52777
+ utrs: 50318
+ video: 52318
+ wikidata-dev: 50084
+ wikidata-topicmaps: 51980
+ wikidumpparse: 51824
+ wikisource-tools: 50336
+ wmt: 52488
+
+# A root must be defined here for every cluster
public:
- scratch: '/exp/scratch *(rw,sec=sys,sync,no_subtree_check,root_squash)'
- root: '/exp *(ro,fsid=0,sec=sys,sync,subtree_check,root_squash,nocrossmnt)'
-private:
- account-creation-assistance:
- gid: 50088
- mounts:
- project: true
- bots:
- gid: 50064
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
- catgraph:
- gid: 50588
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
- chasetestproject:
- gid: 53308
- mounts:
- project: true
- contributors:
- gid: 50408
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
- cvn:
- gid: 50116
- mounts:
- project: true
- wikidata-query:
- gid: 52354
- mounts:
- scratch: true
- dumps: true
- dumps:
- gid: 50124
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
- editor-engagement:
- gid: 50068
- mounts:
- project: true
- etytree:
- gid: 53231
- mounts:
- dumps: true
- fastcci:
- gid: 50983
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
- huggle:
- gid: 50168
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
- maps:
- gid: 50196
- mounts:
- dumps: true
- scratch: true
- maps: true
- maps-team:
- gid: 52497
- mounts:
- scratch: true
- math:
- gid: 50398
- mounts:
- project: true
- scratch: true
- dumps: true
- mwoffliner:
- gid: 52376
- mounts:
- scratch: true
- openstack:
- gid: 50076
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
- osmit:
- gid: 51933
- mounts:
- dumps: true
- project-proxy:
- gid: 50254
- mounts:
- project: true
- quarry:
- gid: 52243
- mounts:
- project: true
- snuggle:
- gid: 50470
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
- testlabs:
- gid: 50302
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
- tools:
- gid: 50380
- mounts:
- dumps: true
- scratch: true
- tools-home: true
- tools-project: true
- toolsbeta:
- gid: 50610
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
- twl:
- gid: 52777
- mounts:
- project: true
- utrs:
- gid: 50318
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
- video:
- gid: 52318
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
- wikidata-dev:
- gid: 50084
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
- wikidata-topicmaps:
- gid: 51980
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
- wikidumpparse:
- gid: 51824
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
- wikisource-tools:
- gid: 50336
- mounts:
- dumps: true
- home: true
- project: true
- scratch: true
- wmt:
- gid: 52488
- mounts:
- project: true
+ secondary:
+ root: '/exp *(ro,fsid=0,sec=sys,sync,subtree_check,root_squash,nocrossmnt)'
+ primary:
+ root: '/exp *(ro,fsid=0,sec=sys,sync,subtree_check,root_squash,nocrossmnt)'
+ #### scratch temporarily moved to labstore1003
+ # scratch: '/exp/scratch *(rw,sec=sys,sync,no_subtree_check,root_squash)'
+
+# Determines what can trigger a per project entry in exports.d/
+# even though we /end up mounting subpaths anyway/
+allowed_private_mounts:
+ - project
+ - home
+ - tools-home
+ - tools-project
+ - maps
+
+# Enabled mounts as read by both server and client side
+
+# Project additions to this file will not be active until 'nfs-manage-binds -f'
+# is run on the active server on the relevant cluster post addition.
+mounts:
+ secondary:
+ account-creation-assistance:
+ - project
+ bots:
+ - project
+ - home
+ catgraph:
+ - project
+ - home
+ chasetestproject:
+ - project
+ contributors:
+ - project
+ - home
+ cvn:
+ - project
+ wikidata-query:
+ - dumps
+ dumps:
+ - project
+ - home
+ editor-engagement:
+ - project
+ fastcci:
+ - project
+ - home
+ huggle:
+ - project
+ - home
+ math:
+ - project
+ openstack:
+ - project
+ - home
+ project-proxy:
+ - project
+ quarry:
+ - project
+ snuggle:
+ - project
+ - home
+ testlabs:
+ - project
+ - home
+ tools:
+ - tools-home
+ - tools-project
+ toolsbeta:
+ - project
+ - scratch
+ twl:
+ - project
+ utrs:
+ - home
+ - project
+ video:
+ - home
+ - project
+ wikidata-dev:
+ - home
+ - project
+ wikidata-topicmaps:
+ - home
+ - project
+ wikidumpparse:
+ - home
+ - project
+ wikisource-tools:
+ - home
+ - project
+ wmt:
+ - project
+ tertiary:
+ bots:
+ - dumps
+ - scratch
+ catgraph:
+ - dumps
+ - scratch
+ contributors:
+ - dumps
+ - scratch
+ wikidata-query:
+ - dumps
+ - scratch
+ dumps:
+ - dumps
+ - scratch
+ etytree:
+ - dumps
+ fastcci:
+ - dumps
+ - scratch
+ huggle:
+ - dumps
+ - scratch
+ maps:
+ - maps
+ - dumps
+ - scratch
+ maps-team:
+ - scratch
+ math:
+ - dumps
+ - scratch
+ mwoffliner:
+ - scratch
+ openstack:
+ - dumps
+ - scratch
+ osmit:
+ - dumps
+ snuggle:
+ - dumps
+ - scratch
+ testlabs:
+ - dumps
+ - scratch
+ tools:
+ - dumps
+ - scratch
+ toolsbeta:
+ - dumps
+ utrs:
+ - dumps
+ - scratch
+ video:
+ - dumps
+ - scratch
+ wikidata-dev:
+ - dumps
+ - scratch
+ wikidata-topicmaps:
+ - dumps
+ - scratch
+ wikidumpparse:
+ - dumps
+ - scratch
+ wikisource-tools:
+ - dumps
+ - scratch
diff --git a/modules/labstore/lib/puppet/parser/functions/mount_nfs_volume.rb
b/modules/labstore/lib/puppet/parser/functions/mount_nfs_volume.rb
index 5f6f853..f46fd56 100644
--- a/modules/labstore/lib/puppet/parser/functions/mount_nfs_volume.rb
+++ b/modules/labstore/lib/puppet/parser/functions/mount_nfs_volume.rb
@@ -5,7 +5,7 @@
# Returns true if the mount should be mounted in
# instance of the project
#
-# Reads this information from labstore/files/nfs-mounts-config.yaml
+# Reads this information from labstore/files/nfs-mounts.yaml
# in the openstack module of operations/puppet.git
module Puppet::Parser::Functions
@@labs_nfs_config_touched = nil
@@ -18,14 +18,20 @@
@@labs_nfs_config = function_loadyaml([path])
@@labs_nfs_config_touched = mtime
end
- config = @@labs_nfs_config['private']
+
project = args[0]
mount = args[1]
- if config.key?(project) && config[project].key?('mounts') \
- && config[project]['mounts'].key?(mount)
- config[project]['mounts'][mount]
- else
- false
+
+ clusters = @@labs_nfs_config['clusters']
+ clusters.each do |cluster, hosts|
+ if !labs_nfs_config['mounts'].key?(cluster)
+ next
+ end
+ cmounts = labs_nfs_config['mounts'][cluster]
+ if cmounts.key?(project) && cmounts[project].include?(mount)
+ return true
+ end
end
+ return false
end
end
--
To view, visit https://gerrit.wikimedia.org/r/345631
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Icbc20ec86ea7057df2262350d8c8d1a052f8c8ad
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Rush <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
