Author: cutting
Date: Wed Feb 5 00:09:57 2014
New Revision: 1564571
URL: http://svn.apache.org/r1564571
Log:
AVRO-1449. Java: Optionally validate default values while reading schemas.
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java
avro/trunk/lang/java/compiler/src/test/idl/input/interop.avdl
avro/trunk/lang/java/compiler/src/test/idl/input/simple.avdl
avro/trunk/lang/java/compiler/src/test/idl/output/interop.avpr
avro/trunk/lang/java/compiler/src/test/idl/output/simple.avpr
avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java
avro/trunk/share/test/data/schema-tests.txt
Modified: avro/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1564571&r1=1564570&r2=1564571&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Wed Feb 5 00:09:57 2014
@@ -18,6 +18,9 @@ Trunk (not yet released)
AVRO-1447. Java: Remove dead code from example in documentation.
(Jesse Anderson via cutting)
+ AVRO-1449. Java: Optionally validate default values while reading schemas.
+ (cutting)
+
BUG FIXES
AVRO-1446. C#: Correctly handle system errors in RPC.
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java?rev=1564571&r1=1564570&r2=1564571&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java
(original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java Wed Feb
5 00:09:57 2014
@@ -372,7 +372,7 @@ public abstract class Schema extends Jso
this.name = validateName(name);
this.schema = schema;
this.doc = doc;
- this.defaultValue = defaultValue;
+ this.defaultValue = validateDefault(name, schema, defaultValue);
this.order = order;
}
public String name() { return name; };
@@ -890,6 +890,7 @@ public abstract class Schema extends Jso
public static class Parser {
private Names names = new Names();
private boolean validate = true;
+ private boolean validateDefaults = false;
/** Adds the provided types to the set of defined, named types known to
* this parser. */
@@ -916,6 +917,15 @@ public abstract class Schema extends Jso
/** True iff names are validated. True by default. */
public boolean getValidate() { return this.validate; }
+ /** Enable or disable default value validation. */
+ public Parser setValidateDefaults(boolean validateDefaults) {
+ this.validateDefaults = validateDefaults;
+ return this;
+ }
+
+ /** True iff default values are validated. False by default. */
+ public boolean getValidateDefaults() { return this.validateDefaults; }
+
/** Parse a schema from the provided file.
* If named, the schema is added to the names known to this parser. */
public Schema parse(File file) throws IOException {
@@ -948,13 +958,16 @@ public abstract class Schema extends Jso
private Schema parse(JsonParser parser) throws IOException {
boolean saved = validateNames.get();
+ boolean savedValidateDefaults = VALIDATE_DEFAULTS.get();
try {
validateNames.set(validate);
+ VALIDATE_DEFAULTS.set(validateDefaults);
return Schema.parse(MAPPER.readTree(parser), names);
} catch (JsonParseException e) {
throw new SchemaParseException(e);
} finally {
validateNames.set(saved);
+ VALIDATE_DEFAULTS.set(savedValidateDefaults);
}
}
}
@@ -1070,6 +1083,75 @@ public abstract class Schema extends Jso
return name;
}
+ private static final ThreadLocal<Boolean> VALIDATE_DEFAULTS
+ = new ThreadLocal<Boolean>() {
+ @Override protected Boolean initialValue() {
+ return false;
+ }
+ };
+
+ private static JsonNode validateDefault(String fieldName, Schema schema,
+ JsonNode defaultValue) {
+ if ((defaultValue != null)
+ && !isValidDefault(schema, defaultValue)) { // invalid default
+ String message = "Invalid default for field "+fieldName
+ +": "+defaultValue+" not a "+schema;
+ if (VALIDATE_DEFAULTS.get())
+ throw new AvroTypeException(message); // throw exception
+ System.err.println("[WARNING] Avro: "+message); // or log warning
+ }
+ return defaultValue;
+ }
+
+ private static boolean isValidDefault(Schema schema, JsonNode defaultValue) {
+ if (defaultValue == null)
+ return false;
+ switch (schema.getType()) {
+ case STRING:
+ case BYTES:
+ case ENUM:
+ case FIXED:
+ return defaultValue.isTextual();
+ case INT:
+ case LONG:
+ case FLOAT:
+ case DOUBLE:
+ return defaultValue.isNumber();
+ case BOOLEAN:
+ return defaultValue.isBoolean();
+ case NULL:
+ return defaultValue.isNull();
+ case ARRAY:
+ if (!defaultValue.isArray())
+ return false;
+ for (JsonNode element : defaultValue)
+ if (!isValidDefault(schema.getElementType(), element))
+ return false;
+ return true;
+ case MAP:
+ if (!defaultValue.isObject())
+ return false;
+ for (JsonNode value : defaultValue)
+ if (!isValidDefault(schema.getValueType(), value))
+ return false;
+ return true;
+ case UNION: // union default: first
branch
+ return isValidDefault(schema.getTypes().get(0), defaultValue);
+ case RECORD:
+ if (!defaultValue.isObject())
+ return false;
+ for (Field field : schema.getFields())
+ if (!isValidDefault(field.schema(),
+ defaultValue.has(field.name())
+ ? defaultValue.get(field.name())
+ : field.defaultValue()))
+ return false;
+ return true;
+ default:
+ return false;
+ }
+ }
+
/** @see #parse(String) */
static Schema parse(JsonNode schema, Names names) {
if (schema.isTextual()) { // name
Modified: avro/trunk/lang/java/compiler/src/test/idl/input/interop.avdl
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/test/idl/input/interop.avdl?rev=1564571&r1=1564570&r2=1564571&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/test/idl/input/interop.avdl (original)
+++ avro/trunk/lang/java/compiler/src/test/idl/input/interop.avdl Wed Feb 5
00:09:57 2014
@@ -39,7 +39,7 @@ protocol InteropProtocol {
float floatField = 0.0;
double doubleField = -1.0e12;
null nullField;
- array<double> arrayField = [{"label":"foo", "children":[]}];
+ array<double> arrayField = [];
map<Foo> mapField;
union { boolean, double, array<bytes> } unionFIeld;
Kind enumField;
Modified: avro/trunk/lang/java/compiler/src/test/idl/input/simple.avdl
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/test/idl/input/simple.avdl?rev=1564571&r1=1564570&r2=1564571&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/test/idl/input/simple.avdl (original)
+++ avro/trunk/lang/java/compiler/src/test/idl/input/simple.avdl Wed Feb 5
00:09:57 2014
@@ -41,9 +41,9 @@ protocol Simple {
/** The kind of record. */
Kind @order("descending") kind;
- @foo("bar") MD5 hash;
+ @foo("bar") MD5 hash = "0000000000000000";
- union { MD5, null} @aliases(["hash", "hsh"]) nullableHash;
+ union {null, MD5} @aliases(["hash", "hsh"]) nullableHash = null;
double value = NaN;
float average = -Infinity;
@@ -55,7 +55,7 @@ protocol Simple {
/** method 'hello' takes @parameter 'greeting' */
string hello(string greeting);
- TestRecord echo(TestRecord `record` = {"name": "bar"});
+ TestRecord echo(TestRecord `record` = {"name":"bar","kind":"BAR"});
/** method 'add' takes @parameter 'arg1' @parameter 'arg2' */
@specialProp("test")
int add(int arg1, int arg2 = 0);
Modified: avro/trunk/lang/java/compiler/src/test/idl/output/interop.avpr
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/test/idl/output/interop.avpr?rev=1564571&r1=1564570&r2=1564571&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/test/idl/output/interop.avpr (original)
+++ avro/trunk/lang/java/compiler/src/test/idl/output/interop.avpr Wed Feb 5
00:09:57 2014
@@ -66,10 +66,7 @@
"type" : "array",
"items" : "double"
},
- "default" : [ {
- "label" : "foo",
- "children" : [ ]
- } ]
+ "default" : [ ]
}, {
"name" : "mapField",
"type" : {
@@ -93,6 +90,5 @@
"type" : "Node"
} ]
} ],
- "messages" : {
- }
-}
+ "messages" : { }
+}
\ No newline at end of file
Modified: avro/trunk/lang/java/compiler/src/test/idl/output/simple.avpr
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/test/idl/output/simple.avpr?rev=1564571&r1=1564570&r2=1564571&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/test/idl/output/simple.avpr (original)
+++ avro/trunk/lang/java/compiler/src/test/idl/output/simple.avpr Wed Feb 5
00:09:57 2014
@@ -31,10 +31,12 @@
"order" : "descending"
}, {
"name" : "hash",
- "type" : "MD5"
+ "type" : "MD5",
+ "default" : "0000000000000000"
}, {
"name" : "nullableHash",
- "type" : [ "MD5", "null" ],
+ "type" : [ "null", "MD5" ],
+ "default" : null,
"aliases" : [ "hash", "hsh" ]
}, {
"name" : "value",
@@ -70,7 +72,8 @@
"name" : "record",
"type" : "TestRecord",
"default" : {
- "name" : "bar"
+ "name" : "bar",
+ "kind" : "BAR"
}
} ],
"response" : "TestRecord"
Modified: avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java?rev=1564571&r1=1564570&r2=1564571&view=diff
==============================================================================
--- avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java
(original)
+++ avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java Wed
Feb 5 00:09:57 2014
@@ -324,16 +324,18 @@ public class TestSchema {
// test that erroneous default values cause errors
for (String type : new String[]
{"int", "long", "float", "double", "string", "bytes", "boolean"}) {
+ checkValidateDefaults("[\""+type+"\", \"null\"]", "null"); // schema
parse time
boolean error = false;
try {
- checkDefault("[\""+type+"\", \"null\"]", "null", 0);
+ checkDefault("[\""+type+"\", \"null\"]", "null", 0); // read time
} catch (AvroTypeException e) {
error = true;
}
assertTrue(error);
+ checkValidateDefaults("[\"null\", \""+type+"\"]", "0"); // schema parse
time
error = false;
try {
- checkDefault("[\"null\", \""+type+"\"]", "0", null);
+ checkDefault("[\"null\", \""+type+"\"]", "0", null); // read time
} catch (AvroTypeException e) {
error = true;
}
@@ -820,6 +822,21 @@ public class TestSchema {
assertEquals("Wrong toString", expected,
Schema.parse(expected.toString()));
}
+ private static void checkValidateDefaults(String schemaJson, String
defaultJson) {
+ try {
+ Schema.Parser parser = new Schema.Parser();
+ parser.setValidateDefaults(true);
+ String recordJson =
+ "{\"type\":\"record\", \"name\":\"Foo\",
\"fields\":[{\"name\":\"f\", "
+ +"\"type\":"+schemaJson+", "
+ +"\"default\":"+defaultJson+"}]}";
+ parser.parse(recordJson);
+ fail("Schema of type " + schemaJson + " should not have default " +
defaultJson);
+ } catch (AvroTypeException e) {
+ return;
+ }
+ }
+
@Test(expected=AvroTypeException.class)
public void testNoDefaultField() throws Exception {
Schema expected =
Modified: avro/trunk/share/test/data/schema-tests.txt
URL:
http://svn.apache.org/viewvc/avro/trunk/share/test/data/schema-tests.txt?rev=1564571&r1=1564570&r2=1564571&view=diff
==============================================================================
--- avro/trunk/share/test/data/schema-tests.txt (original)
+++ avro/trunk/share/test/data/schema-tests.txt Wed Feb 5 00:09:57 2014
@@ -144,7 +144,7 @@
// 026
<<INPUT
-{ "fields":[{"type":"boolean", "aliases":[], "name":"f1", "default":"true"},
+{ "fields":[{"type":"boolean", "aliases":[], "name":"f1", "default":true},
{"order":"descending","name":"f2","doc":"Hello","type":"int"}],
"type":"record", "name":"foo"
}
