This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new 3a7786e AVRO-2200: Option to fail when extra fields are in the
payload (#321)
3a7786e is described below
commit 3a7786ed1a3bedf05e6bce7924aea2f75b576912
Author: Daniel Orner <[email protected]>
AuthorDate: Wed Apr 3 04:45:19 2019 -0400
AVRO-2200: Option to fail when extra fields are in the payload (#321)
* AVRO-2200: Option to fail when extra fields are in the payload
* Check for extra fields recursively
---
lang/ruby/lib/avro/schema_validator.rb | 35 +++++++------
lang/ruby/test/test_schema_validator.rb | 87 ++++++++++++++++++++++++++++++++-
2 files changed, 106 insertions(+), 16 deletions(-)
diff --git a/lang/ruby/lib/avro/schema_validator.rb
b/lang/ruby/lib/avro/schema_validator.rb
index 485092c..f9274f0 100644
--- a/lang/ruby/lib/avro/schema_validator.rb
+++ b/lang/ruby/lib/avro/schema_validator.rb
@@ -63,7 +63,7 @@ module Avro
TypeMismatchError = Class.new(ValidationError)
class << self
- def validate!(expected_schema, logical_datum, options = { recursive:
true, encoded: false })
+ def validate!(expected_schema, logical_datum, options = { recursive:
true, encoded: false, fail_on_extra_fields: false })
options ||= {}
options[:recursive] = true unless options.key?(:recursive)
@@ -86,16 +86,23 @@ module Avro
case expected_schema.type_sym
when :array
- validate_array(expected_schema, datum, path, result)
+ validate_array(expected_schema, datum, path, result, options)
when :map
- validate_map(expected_schema, datum, path, result)
+ validate_map(expected_schema, datum, path, result, options)
when :union
- validate_union(expected_schema, datum, path, result)
+ validate_union(expected_schema, datum, path, result, options)
when :record, :error, :request
fail TypeMismatchError unless datum.is_a?(Hash)
expected_schema.fields.each do |field|
deeper_path = deeper_path_for_hash(field.name, path)
- validate_recursive(field.type, datum[field.name], deeper_path,
result)
+ validate_recursive(field.type, datum[field.name], deeper_path,
result, options)
+ end
+ if options[:fail_on_extra_fields]
+ datum_fields = datum.keys.map(&:to_s)
+ schema_fields = expected_schema.fields.map(&:name)
+ (datum_fields - schema_fields).each do |extra_field|
+ result.add_error(path, "extra field '#{extra_field}' - not in
schema")
+ end
end
end
rescue TypeMismatchError
@@ -156,31 +163,31 @@ module Avro
"expected enum with values #{symbols}, got
#{actual_value_message(datum)}"
end
- def validate_array(expected_schema, datum, path, result)
+ def validate_array(expected_schema, datum, path, result, options = {})
fail TypeMismatchError unless datum.is_a?(Array)
datum.each_with_index do |d, i|
- validate_recursive(expected_schema.items, d, path + "[#{i}]", result)
+ validate_recursive(expected_schema.items, d, path + "[#{i}]",
result, options)
end
end
- def validate_map(expected_schema, datum, path, result)
+ def validate_map(expected_schema, datum, path, result, options = {})
fail TypeMismatchError unless datum.is_a?(Hash)
datum.keys.each do |k|
result.add_error(path, "unexpected key type
'#{ruby_to_avro_type(k.class)}' in map") unless k.is_a?(String)
end
datum.each do |k, v|
deeper_path = deeper_path_for_hash(k, path)
- validate_recursive(expected_schema.values, v, deeper_path, result)
+ validate_recursive(expected_schema.values, v, deeper_path, result,
options)
end
end
- def validate_union(expected_schema, datum, path, result)
+ def validate_union(expected_schema, datum, path, result, options = {})
if expected_schema.schemas.size == 1
- validate_recursive(expected_schema.schemas.first, datum, path,
result)
+ validate_recursive(expected_schema.schemas.first, datum, path,
result, options)
return
end
failures = []
- compatible_type = first_compatible_type(datum, expected_schema, path,
failures)
+ compatible_type = first_compatible_type(datum, expected_schema, path,
failures, options)
return unless compatible_type.nil?
complex_type_failed = failures.detect { |r|
COMPLEX_TYPES.include?(r[:type]) }
@@ -192,10 +199,10 @@ module Avro
end
end
- def first_compatible_type(datum, expected_schema, path, failures)
+ def first_compatible_type(datum, expected_schema, path, failures,
options = {})
expected_schema.schemas.find do |schema|
result = Result.new
- validate_recursive(schema, datum, path, result)
+ validate_recursive(schema, datum, path, result, options)
failures << { type: schema.type_sym, result: result } if
result.failure?
!result.failure?
end
diff --git a/lang/ruby/test/test_schema_validator.rb
b/lang/ruby/test/test_schema_validator.rb
index 9b74d61..0126c35 100644
--- a/lang/ruby/test/test_schema_validator.rb
+++ b/lang/ruby/test/test_schema_validator.rb
@@ -17,8 +17,8 @@
require 'test_help'
class TestSchema < Test::Unit::TestCase
- def validate!(schema, value)
- Avro::SchemaValidator.validate!(schema, value)
+ def validate!(schema, value, options=nil)
+ Avro::SchemaValidator.validate!(schema, value, options)
end
def validate_simple!(schema, value)
@@ -468,4 +468,87 @@ class TestSchema < Test::Unit::TestCase
exception.to_s
)
end
+
+ def test_validate_extra_fields
+ schema = hash_to_schema(
+ type: 'record',
+ name: 'fruits',
+ fields: [
+ {
+ name: 'veggies',
+ type: 'string'
+ }
+ ]
+ )
+ exception = assert_raise(Avro::SchemaValidator::ValidationError) do
+ validate!(schema, {'veggies' => 'tomato', 'bread' => 'rye'},
fail_on_extra_fields: true)
+ end
+ assert_equal(1, exception.result.errors.size)
+ assert_equal("at . extra field 'bread' - not in schema",
+ exception.to_s)
+ end
+
+ def test_validate_subrecord_extra_fields
+ schema = hash_to_schema(type: 'record',
+ name: 'top',
+ fields: [
+ {
+ name: 'fruit',
+ type: {
+ name: 'fruit',
+ type: 'record',
+ fields: [{ name: 'name', type: 'string' }]
+ }
+ }
+ ])
+ exception = assert_raise(Avro::SchemaValidator::ValidationError) do
+ validate!(schema, { 'fruit' => { 'name' => 'orange', 'color' => 'orange'
} }, fail_on_extra_fields: true)
+ end
+ assert_equal(1, exception.result.errors.size)
+ assert_equal("at .fruit extra field 'color' - not in schema",
exception.to_s)
+ end
+
+ def test_validate_array_extra_fields
+ schema = hash_to_schema(type: 'array',
+ items: {
+ name: 'fruit',
+ type: 'record',
+ fields: [{ name: 'name', type: 'string' }]
+ })
+ exception = assert_raise(Avro::SchemaValidator::ValidationError) do
+ validate!(schema, [{ 'name' => 'orange', 'color' => 'orange' }],
fail_on_extra_fields: true)
+ end
+ assert_equal(1, exception.result.errors.size)
+ assert_equal("at .[0] extra field 'color' - not in schema", exception.to_s)
+ end
+
+ def test_validate_map_extra_fields
+ schema = hash_to_schema(type: 'map',
+ values: {
+ name: 'fruit',
+ type: 'record',
+ fields: [{ name: 'color', type: 'string' }]
+ })
+ exception = assert_raise(Avro::SchemaValidator::ValidationError) do
+ validate!(schema, { 'apple' => { 'color' => 'green', 'extra' => 1 } },
fail_on_extra_fields: true)
+ end
+ assert_equal(1, exception.result.errors.size)
+ assert_equal("at .apple extra field 'extra' - not in schema",
exception.to_s)
+ end
+
+ def test_validate_union_extra_fields
+ schema = hash_to_schema([
+ 'null',
+ {
+ type: 'record',
+ name: 'fruit',
+ fields: [{ name: 'name', type: 'string' }]
+ }
+ ])
+ exception = assert_raise(Avro::SchemaValidator::ValidationError) do
+ validate!(schema, { 'name' => 'apple', 'color' => 'green' },
fail_on_extra_fields: true)
+ end
+ assert_equal(1, exception.result.errors.size)
+ assert_equal("at . extra field 'color' - not in schema", exception.to_s)
+ end
end
