Gehel has submitted this change and it was merged. Change subject: Expose elasticsearch through HTTP ......................................................................
Expose elasticsearch through HTTP Adding nginx to do SSL termination, allowing clients to use either HTTP or HTTPS at will. Puppet certificates are used here, exposed through base::expose_puppet_certs. This makes this change dependant on https://gerrit.wikimedia.org/r/#/c/274382/ SSL certificates mush match the service name used to access elasticsearch. For this we configure base::puppet:dns_alt_names and we need to recreate puppet SSL certs (recreation of certs is done manually). Bug: T124444 Change-Id: I347caf322ee17876978ae8c0d94b3e38a6102cdd --- M hieradata/labs/deployment-prep/common.yaml M hieradata/labs/deployment-prep/host/deployment-logstash2.yaml M hieradata/role/codfw/elasticsearch/server.yaml M hieradata/role/common/elasticsearch/server.yaml M hieradata/role/common/logstash/elasticsearch.yaml M hieradata/role/eqiad/elasticsearch/server.yaml A modules/elasticsearch/manifests/https.pp M modules/elasticsearch/manifests/init.pp A modules/elasticsearch/spec/defines/https_rspec.rb A modules/elasticsearch/templates/nginx/es-ssl-termination.nginx.conf.erb 10 files changed, 91 insertions(+), 0 deletions(-) Approvals: Gehel: Looks good to me, approved EBernhardson: Looks good to me, but someone else must approve jenkins-bot: Verified diff --git a/hieradata/labs/deployment-prep/common.yaml b/hieradata/labs/deployment-prep/common.yaml index f9b945c..8884e83 100644 --- a/hieradata/labs/deployment-prep/common.yaml +++ b/hieradata/labs/deployment-prep/common.yaml @@ -177,6 +177,7 @@ "elasticsearch::auto_create_index": '+apifeatureusage-*,-*' "elasticsearch::graylog_hosts": - deployment-logstash2.deployment-prep.eqiad.wmflabs +"elasticsearch::https::ensure": present role::logstash::statsd_host: labmon1001.eqiad.wmnet "mediawiki::redis_servers::eqiad": shard01: diff --git a/hieradata/labs/deployment-prep/host/deployment-logstash2.yaml b/hieradata/labs/deployment-prep/host/deployment-logstash2.yaml index 766d4fc..c6835aa 100644 --- a/hieradata/labs/deployment-prep/host/deployment-logstash2.yaml +++ b/hieradata/labs/deployment-prep/host/deployment-logstash2.yaml @@ -13,6 +13,7 @@ # Needed to keep from trying to join cirrussearch cluster elasticsearch::unicast_hosts: - deployment-logstash2.deployment-prep.eqiad.wmflabs +elasticsearch::https::ensure: absent elasticsearch::auto_create_index: '+logstash-*,-*' # Logstash specific diff --git a/hieradata/role/codfw/elasticsearch/server.yaml b/hieradata/role/codfw/elasticsearch/server.yaml index c7cecf8..61b84df 100644 --- a/hieradata/role/codfw/elasticsearch/server.yaml +++ b/hieradata/role/codfw/elasticsearch/server.yaml @@ -45,3 +45,5 @@ - logstash1001.eqiad.wmnet - logstash1002.eqiad.wmnet - logstash1003.eqiad.wmnet + +base::puppet::dns_alt_names: 'search.svc.codfw.wmnet' diff --git a/hieradata/role/common/elasticsearch/server.yaml b/hieradata/role/common/elasticsearch/server.yaml index e8a38da..af62c2b 100644 --- a/hieradata/role/common/elasticsearch/server.yaml +++ b/hieradata/role/common/elasticsearch/server.yaml @@ -34,3 +34,5 @@ # Use only 1 merge thread (instead of 3) to avoid updates interfering with # actual searches elasticsearch::merge_threads: 1 + +elasticsearch::https::ensure: 'present' diff --git a/hieradata/role/common/logstash/elasticsearch.yaml b/hieradata/role/common/logstash/elasticsearch.yaml index 64f5858..0e7d493 100644 --- a/hieradata/role/common/logstash/elasticsearch.yaml +++ b/hieradata/role/common/logstash/elasticsearch.yaml @@ -29,3 +29,5 @@ debdeploy::grains: debdeploy-logstash: value: standard + +elasticsearch::https::ensure: 'absent' diff --git a/hieradata/role/eqiad/elasticsearch/server.yaml b/hieradata/role/eqiad/elasticsearch/server.yaml index 5b0fdb9..58bff9f 100644 --- a/hieradata/role/eqiad/elasticsearch/server.yaml +++ b/hieradata/role/eqiad/elasticsearch/server.yaml @@ -47,3 +47,10 @@ debdeploy::grains: debdeploy-elastic-eqiad: value: standard + +elasticsearch::graylog_hosts: + - logstash1001.eqiad.wmnet + - logstash1002.eqiad.wmnet + - logstash1003.eqiad.wmnet + +base::puppet::dns_alt_names: 'search.svc.eqiad.wmnet' diff --git a/modules/elasticsearch/manifests/https.pp b/modules/elasticsearch/manifests/https.pp new file mode 100644 index 0000000..b9220c2 --- /dev/null +++ b/modules/elasticsearch/manifests/https.pp @@ -0,0 +1,33 @@ +# = Class: elasticsearch::https +# +# This class configures HTTPS for elasticsearch +# +# == Parameters: +# - ensure: self explanatory +class elasticsearch::https ( + $ensure = 'absent', +){ + + class { 'nginx::ssl': + ensure => $ensure, + } + + ::base::expose_puppet_certs { '/etc/nginx': + ensure => $ensure, + provide_private => true, + ssldir => '/var/lib/puppet/client/ssl', + } + + ::nginx::site { 'elasticsearch-ssl-termination': + ensure => $ensure, + content => template('elasticsearch/nginx/es-ssl-termination.nginx.conf.erb'), + } + + ::ferm::service { 'elastic-https': + ensure => $ensure, + proto => 'tcp', + port => '9243', + srange => '$INTERNAL', + } + +} diff --git a/modules/elasticsearch/manifests/init.pp b/modules/elasticsearch/manifests/init.pp index 53e643b..80eb707 100644 --- a/modules/elasticsearch/manifests/init.pp +++ b/modules/elasticsearch/manifests/init.pp @@ -115,6 +115,7 @@ } include ::elasticsearch::packages + include ::elasticsearch::https file { '/etc/elasticsearch/elasticsearch.yml': ensure => file, diff --git a/modules/elasticsearch/spec/defines/https_rspec.rb b/modules/elasticsearch/spec/defines/https_rspec.rb new file mode 100644 index 0000000..6bf3608 --- /dev/null +++ b/modules/elasticsearch/spec/defines/https_rspec.rb @@ -0,0 +1,22 @@ +require 'spec_helper' + +describe 'elasticsearch::https', :type => :class do + let(:facts) { { :lsbdistrelease => 'ubuntu', + :lsbdistid => 'trusty', + :fqdn => 'host.example.net' + } } + + describe 'certificates are absent by default' do + it { should contain_file('/etc/nginx/ssl/cert.pem').with({ 'ensure' => 'absent' }) } + it { should contain_file('/etc/nginx/ssl/server.key').with({ 'ensure' => 'absent' }) } + end + + describe 'When enabled, nginx is installed and certificates are available' do + let(:params) { { :ensure => 'present' } } + + it { should contain_package('nginx-full') } + it { should contain_file('/etc/nginx/ssl/cert.pem').with({ 'ensure' => 'present' }) } + it { should contain_file('/etc/nginx/ssl/server.key').with({ 'ensure' => 'present' }) } + end + +end diff --git a/modules/elasticsearch/templates/nginx/es-ssl-termination.nginx.conf.erb b/modules/elasticsearch/templates/nginx/es-ssl-termination.nginx.conf.erb new file mode 100644 index 0000000..90158f6 --- /dev/null +++ b/modules/elasticsearch/templates/nginx/es-ssl-termination.nginx.conf.erb @@ -0,0 +1,20 @@ +# This file is managed by puppet. Do not edit. +server { + listen 9243 ssl default_server; + listen [::]:9243 ssl default_server ipv6only=on; + + ssl_certificate /etc/nginx/ssl/cert.pem; + ssl_certificate_key /etc/nginx/ssl/server.key; + + access_log /var/log/nginx/ssl_access.log; + error_log /var/log/nginx/ssl_error.log; + + root /dev/null; + + location / { + proxy_pass http://localhost:9200/; + + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + } +} -- To view, visit https://gerrit.wikimedia.org/r/274711 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I347caf322ee17876978ae8c0d94b3e38a6102cdd Gerrit-PatchSet: 12 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Gehel <[email protected]> Gerrit-Reviewer: DCausse <[email protected]> Gerrit-Reviewer: EBernhardson <[email protected]> Gerrit-Reviewer: Faidon Liambotis <[email protected]> Gerrit-Reviewer: Filippo Giunchedi <[email protected]> Gerrit-Reviewer: Gehel <[email protected]> Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
