EBernhardson has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/344965 )
Change subject: [WIP] Update elk stack to 5.x
......................................................................
[WIP] Update elk stack to 5.x
* Upgrades elasticsearch to 5.1.2
* Upgrades kibana to 5.1.2
* Updates templates used for logstash for compatability with es 5.x
** index is now a boolean, instead of analyzed/not_analyzed/no. New
settings are basically: analyzed => type: text index: true,
not_analyzed => type:keyword index:true, no => index:false
Bug: T154473
Change-Id: I5ee2ace057ef9a7398d7c896fa513fb292050929
---
M hieradata/role/common/logstash/collector.yaml
M modules/role/files/logstash/apifeatureusage-template.json
M modules/role/files/logstash/elasticsearch-template.json
3 files changed, 43 insertions(+), 81 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/65/344965/1
diff --git a/hieradata/role/common/logstash/collector.yaml
b/hieradata/role/common/logstash/collector.yaml
index ca43eeb..dbb8e2a 100644
--- a/hieradata/role/common/logstash/collector.yaml
+++ b/hieradata/role/common/logstash/collector.yaml
@@ -58,3 +58,6 @@
debdeploy-logstash:
value: standard
+# install elasticsearch 5.x and kibana 5.x, which are
+# only available in experimental at the moment.
+apt::use_experimental: true
diff --git a/modules/role/files/logstash/apifeatureusage-template.json
b/modules/role/files/logstash/apifeatureusage-template.json
index 2228152..37fd1dd 100644
--- a/modules/role/files/logstash/apifeatureusage-template.json
+++ b/modules/role/files/logstash/apifeatureusage-template.json
@@ -21,15 +21,15 @@
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
- "index" : "no"
+ "index" : false
}
}
} ],
"properties" : {
- "@timestamp" : { "type" : "date", "index" : "not_analyzed" },
- "@version" : { "type" : "string", "index" : "not_analyzed" },
- "feature" : { "type" : "string", "index" : "not_analyzed" },
- "agent" : { "type" : "string", "index" : "not_analyzed" }
+ "@timestamp" : { "type" : "date", "index" : true },
+ "@version" : { "type" : "keyword", "index" : true },
+ "feature" : { "type" : "keyword", "index" : true },
+ "agent" : { "type" : "keyword", "index" : true }
}
}
}
diff --git a/modules/role/files/logstash/elasticsearch-template.json
b/modules/role/files/logstash/elasticsearch-template.json
index 21e909f..0b8643f 100644
--- a/modules/role/files/logstash/elasticsearch-template.json
+++ b/modules/role/files/logstash/elasticsearch-template.json
@@ -3,7 +3,6 @@
"settings" : {
"number_of_shards" : 1,
"number_of_replicas" : 2,
- "index.cache.field.type" : "soft",
"index.refresh_interval" : "5s",
"analysis" : {
"analyzer" : {
@@ -26,19 +25,15 @@
{
"string_fields" : {
"match" : "*",
- "match_mapping_type" : "string",
+ "match_mapping_type" : "text",
"mapping" : {
- "type" : "multi_field",
+ "type" : "text",
+ "index" : true,
+ "omit_norms" : true
"fields" : {
- "{name}" : {
- "type" : "string",
- "index" : "analyzed",
- "omit_norms" : true
- },
"raw" : {
- "type" : "string",
- "index" : "not_analyzed",
- "ignore_above" : 256,
+ "type" : "keyword",
+ "index" : true,
"doc_values": true
}
}
@@ -49,20 +44,9 @@
"boolean_to_string" : {
"match_mapping_type" : "boolean",
"mapping" : {
- "type" : "multi_field",
- "fields" : {
- "{name}" : {
- "type" : "string",
- "index" : "analyzed",
- "omit_norms" : true
- },
- "raw" : {
- "type" : "string",
- "index" : "not_analyzed",
- "ignore_above" : 256,
- "doc_values": true
- }
- }
+ "type" : "keyword",
+ "index" : true,
+ "doc_values": true
}
}
},
@@ -70,20 +54,9 @@
"double_to_string" : {
"match_mapping_type" : "double",
"mapping" : {
- "type" : "multi_field",
- "fields" : {
- "{name}" : {
- "type" : "string",
- "index" : "analyzed",
- "omit_norms" : true
- },
- "raw" : {
- "type" : "string",
- "index" : "not_analyzed",
- "ignore_above" : 256,
- "doc_values": true
- }
- }
+ "type" : "keyword",
+ "index" : true,
+ "doc_values": true
}
}
},
@@ -91,20 +64,9 @@
"long_to_string" : {
"match_mapping_type" : "long",
"mapping" : {
- "type" : "multi_field",
- "fields" : {
- "{name}" : {
- "type" : "string",
- "index" : "analyzed",
- "omit_norms" : true
- },
- "raw" : {
- "type" : "string",
- "index" : "not_analyzed",
- "ignore_above" : 256,
- "doc_values": true
- }
- }
+ "type" : "keyword",
+ "index" : true,
+ "doc_values": true
}
}
}
@@ -112,12 +74,12 @@
"properties" : {
"@timestamp" : {
"type" : "date",
- "index" : "not_analyzed",
+ "index" : true,
"doc_values": true
},
"@version" : {
- "type" : "string",
- "index" : "not_analyzed",
+ "type" : "keyword",
+ "index" : true,
"doc_values": true
},
"geoip" : {
@@ -131,32 +93,29 @@
}
},
"host" : {
- "type" : "string",
- "index" : "not_analyzed",
+ "type" : "keyword",
+ "index" : true,
"doc_values": true
},
"level" : {
- "type" : "string",
- "index" : "not_analyzed",
+ "type" : "keyword",
+ "index" : true,
"doc_values": true
},
"message" : {
- "type" : "string",
- "index" : "analyzed",
+ "type" : "text",
+ "index" : true,
"omit_norms" : true
},
"normalized_message" : {
- "type": "multi_field",
+ "type": "text",
+ "index" : true,
+ "omit_norms" : true,
"fields": {
"raw": {
- "type": "string",
- "index": "not_analyzed",
+ "type": "keyword",
+ "index": true,
"doc_values": true
- },
- "normalized_message": {
- "index" : "analyzed",
- "omit_norms" : true,
- "type" : "string"
}
}
},
@@ -165,18 +124,18 @@
"doc_values": true
},
"tags" : {
- "type" : "string",
- "index" : "not_analyzed",
+ "type" : "keyword",
+ "index" : true,
"doc_values": true
},
"type" : {
- "type" : "string",
- "index" : "not_analyzed",
+ "type" : "keyword",
+ "index" : true,
"doc_values": true
},
"udp_sender" : {
- "type" : "string",
- "index" : "not_analyzed",
+ "type" : "keyword",
+ "index" : true,
"doc_values": true
}
}
--
To view, visit https://gerrit.wikimedia.org/r/344965
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I5ee2ace057ef9a7398d7c896fa513fb292050929
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: EBernhardson <ebernhard...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
