BryanDavis has uploaded a new change for review. https://gerrit.wikimedia.org/r/229073
Change subject: Add logstash-filter-prune 0.1.5 ...................................................................... Add logstash-filter-prune 0.1.5 Abandon the idea of using gitfat to ship raw gems and instead just do the simplest think possible which is to unpack the needed gem into this repo and then configure logstash to source it using the `--pluginpath` directive. Thankfully upstream came to their senses and restored this functionality in 1.5.3 after removing it in 1.5.0 (<https://github.com/elastic/logstash/pull/3587>). Bug: T99735 Change-Id: Ied616e2e9353567cd2aaffbf57b5114620dcc24f --- D .gitattributes D .gitfat M .gitignore A logstash-filter-prune.gemspec A logstash/filters/prune.rb 5 files changed, 180 insertions(+), 5 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/software/logstash/plugins refs/changes/73/229073/1 diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index d9f744b..0000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -*.gem filter=fat -crlf diff --git a/.gitfat b/.gitfat deleted file mode 100644 index 108e44a..0000000 --- a/.gitfat +++ /dev/null @@ -1,3 +0,0 @@ -[rsync] -remote = archiva.wikimedia.org::archiva/git-fat -options = --copy-links --verbose diff --git a/.gitignore b/.gitignore index 7b18b55..3300a23 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ -.deploy +*.gem +Gemfile.lock +.bundle +vendor diff --git a/logstash-filter-prune.gemspec b/logstash-filter-prune.gemspec new file mode 100644 index 0000000..08b0340 --- /dev/null +++ b/logstash-filter-prune.gemspec @@ -0,0 +1,27 @@ +Gem::Specification.new do |s| + + s.name = 'logstash-filter-prune' + s.version = '0.1.5' + s.licenses = ['Apache License (2.0)'] + s.summary = "The prune filter is for pruning event data from fields based on whitelist/blacklist of field names or their values (names and values can also be regular expressions)" + s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" + s.authors = ["Elastic"] + s.email = '[email protected]' + s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"; + s.require_paths = ["lib"] + + # Files + s.files = `git ls-files`.split($\) + + # Tests + s.test_files = s.files.grep(%r{^(test|spec|features)/}) + + # Special flag to let us know this is actually a logstash plugin + s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" } + + # Gem dependencies + s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0' + + s.add_development_dependency 'logstash-devutils' +end + diff --git a/logstash/filters/prune.rb b/logstash/filters/prune.rb new file mode 100644 index 0000000..d85be39 --- /dev/null +++ b/logstash/filters/prune.rb @@ -0,0 +1,149 @@ +# encoding: utf-8 +require "logstash/filters/base" +require "logstash/namespace" + + +# The prune filter is for pruning event data from `@fields` based on whitelist/blacklist +# of field names or their values (names and values can also be regular expressions). + +class LogStash::Filters::Prune < LogStash::Filters::Base + config_name "prune" + + # Trigger whether configation fields and values should be interpolated for + # dynamic values. + # Probably adds some performance overhead. Defaults to false. + config :interpolate, :validate => :boolean, :default => false + + # Include only fields only if their names match specified regexps, default to empty list which means include everything. + # [source,ruby] + # filter { + # %PLUGIN% { + # tags => [ "apache-accesslog" ] + # whitelist_names => [ "method", "(referrer|status)", "${some}_field" ] + # } + # } + config :whitelist_names, :validate => :array, :default => [] + + # Exclude fields which names match specified regexps, by default exclude unresolved `%{field}` strings. + # [source,ruby] + # filter { + # %PLUGIN% { + # tags => [ "apache-accesslog" ] + # blacklist_names => [ "method", "(referrer|status)", "${some}_field" ] + # } + # } + config :blacklist_names, :validate => :array, :default => [ "%\{[^}]+\}" ] + + # Include specified fields only if their values match regexps. + # In case field values are arrays, the fields are pruned on per array item + # thus only matching array items will be included. + # [source,ruby] + # filter { + # %PLUGIN% { + # tags => [ "apache-accesslog" ] + # whitelist_values => [ "uripath", "/index.php", + # "method", "(GET|POST)", + # "status", "^[^2]" ] + # } + # } + config :whitelist_values, :validate => :hash, :default => {} + + # Exclude specified fields if their values match regexps. + # In case field values are arrays, the fields are pruned on per array item + # in case all array items are matched whole field will be deleted. + # [source,ruby] + # filter { + # %PLUGIN% { + # tags => [ "apache-accesslog" ] + # blacklist_values => [ "uripath", "/index.php", + # "method", "(HEAD|OPTIONS)", + # "status", "^[^2]" ] + # } + # } + config :blacklist_values, :validate => :hash, :default => {} + + public + def register + unless @interpolate + @whitelist_names_regexp = Regexp.union(@whitelist_names.map {|x| Regexp.new(x)}) + @blacklist_names_regexp = Regexp.union(@blacklist_names.map {|x| Regexp.new(x)}) + @whitelist_values.each do |key, value| + @whitelist_values[key] = Regexp.new(value) + end + @blacklist_values.each do |key, value| + @blacklist_values[key] = Regexp.new(value) + end + end + end # def register + + public + def filter(event) + return unless filter?(event) + + hash = event.to_hash + + # We need to collect fields which needs to be remove ,and only in the end + # actually remove it since then interpolation mode you can get unexpected + # results as fields with dynamic values will not match since the fields to + # which they refer have already been removed. + fields_to_remove = [] + + unless @whitelist_names.empty? + @whitelist_names_regexp = Regexp.union(@whitelist_names.map {|x| Regexp.new(event.sprintf(x))}) if @interpolate + hash.each_key do |field| + fields_to_remove << field unless field.match(@whitelist_names_regexp) + end + end + + unless @blacklist_names.empty? + @blacklist_names_regexp = Regexp.union(@blacklist_names.map {|x| Regexp.new(event.sprintf(x))}) if @interpolate + hash.each_key do |field| + fields_to_remove << field if field.match(@blacklist_names_regexp) + end + end + + @whitelist_values.each do |key, value| + if @interpolate + key = event.sprintf(key) + value = Regexp.new(event.sprintf(value)) + end + if hash[key] + if hash[key].is_a?(Array) + subvalues_to_remove = hash[key].find_all{|x| not x.match(value)} + unless subvalues_to_remove.empty? + fields_to_remove << (subvalues_to_remove.length == hash[key].length ? key : { :key => key, :values => subvalues_to_remove }) + end + else + fields_to_remove << key if not hash[key].match(value) + end + end + end + + @blacklist_values.each do |key, value| + if @interpolate + key = event.sprintf(key) + value = Regexp.new(event.sprintf(value)) + end + if hash[key] + if hash[key].is_a?(Array) + subvalues_to_remove = hash[key].find_all{|x| x.match(value)} + unless subvalues_to_remove.empty? + fields_to_remove << (subvalues_to_remove.length == hash[key].length ? key : { :key => key, :values => subvalues_to_remove }) + end + else + fields_to_remove << key if hash[key].match(value) + end + end + end + + fields_to_remove.each do |field| + if field.is_a?(Hash) + hash[field[:key]] = hash[field[:key]] - field[:values] + else + hash.delete(field) + end + end + + filter_matched(event) + end # def filter +end # class LogStash::Filters::Prune -- To view, visit https://gerrit.wikimedia.org/r/229073 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ied616e2e9353567cd2aaffbf57b5114620dcc24f Gerrit-PatchSet: 1 Gerrit-Project: operations/software/logstash/plugins Gerrit-Branch: master Gerrit-Owner: BryanDavis <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
