Repository: avro Updated Branches: refs/heads/master 3af404efb -> 2df0775d2
AVRO-2003: Report specific location of schema incompatibilities Closes #201 Signed-off-by: Nandor Kollar <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/avro/repo Commit: http://git-wip-us.apache.org/repos/asf/avro/commit/2df0775d Tree: http://git-wip-us.apache.org/repos/asf/avro/tree/2df0775d Diff: http://git-wip-us.apache.org/repos/asf/avro/diff/2df0775d Branch: refs/heads/master Commit: 2df0775d2f368b326e3ac6442ce4850e3fe62edc Parents: 3af404e Author: Elliot West <[email protected]> Authored: Tue Nov 21 17:18:01 2017 +0100 Committer: Nandor Kollar <[email protected]> Committed: Tue Nov 21 17:28:28 2017 +0100 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/avro/SchemaCompatibility.java | 478 ++++++++++++------- .../apache/avro/TestSchemaCompatibility.java | 71 ++- ...estSchemaCompatibilityFixedSizeMismatch.java | 14 +- ...stSchemaCompatibilityMissingEnumSymbols.java | 10 +- ...stSchemaCompatibilityMissingUnionBranch.java | 49 +- .../avro/TestSchemaCompatibilityMultiple.java | 158 ++++++ .../TestSchemaCompatibilityNameMismatch.java | 17 +- ...atibilityReaderFieldMissingDefaultValue.java | 9 +- .../TestSchemaCompatibilityTypeMismatch.java | 66 +-- .../test/java/org/apache/avro/TestSchemas.java | 26 +- 11 files changed, 648 insertions(+), 251 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/avro/blob/2df0775d/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index eba79d4..a621c65 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,7 @@ Avro Change Log Trunk (not yet released) INCOMPATIBLE CHANGES + AVRO-2003: Report specific location of schema incompatibilities (teabot via nkollar) AVRO-2035: Java: validate default values when parsing schemas. (cutting) http://git-wip-us.apache.org/repos/asf/avro/blob/2df0775d/lang/java/avro/src/main/java/org/apache/avro/SchemaCompatibility.java ---------------------------------------------------------------------- diff --git a/lang/java/avro/src/main/java/org/apache/avro/SchemaCompatibility.java b/lang/java/avro/src/main/java/org/apache/avro/SchemaCompatibility.java index 4b454f6..aa5041e 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/SchemaCompatibility.java +++ b/lang/java/avro/src/main/java/org/apache/avro/SchemaCompatibility.java @@ -17,8 +17,11 @@ */ package org.apache.avro; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Deque; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -161,22 +164,6 @@ public class SchemaCompatibility { mWriter = writer; } - /** - * Returns the reader schema in this pair. - * @return the reader schema in this pair. - */ - public Schema getReader() { - return mReader; - } - - /** - * Returns the writer schema in this pair. - * @return the writer schema in this pair. - */ - public Schema getWriter() { - return mWriter; - } - /** {@inheritDoc} */ @Override public int hashCode() { @@ -208,6 +195,7 @@ public class SchemaCompatibility { * <p> Provides memoization to handle recursive schemas. </p> */ private static final class ReaderWriterCompatiblityChecker { + private static final String ROOT_REFERENCE_TOKEN = ""; private final Map<ReaderWriter, SchemaCompatibilityResult> mMemoizeMap = new HashMap<ReaderWriter, SchemaCompatibilityResult>(); @@ -224,22 +212,42 @@ public class SchemaCompatibility { final Schema reader, final Schema writer ) { + Deque<String> location = new ArrayDeque<String>(); + return getCompatibility(ROOT_REFERENCE_TOKEN, reader, writer, location); + } + + /** + * Reports the compatibility of a reader/writer schema pair. + * <p> Memoizes the compatibility results. </p> + * @param referenceToken The equivalent JSON pointer reference token representation of the schema node being visited. + * @param reader Reader schema to test. + * @param writer Writer schema to test. + * @param location Stack with which to track the location within the schema. + * @return the compatibility of the reader/writer schema pair. + */ + private SchemaCompatibilityResult getCompatibility( + String referenceToken, + final Schema reader, + final Schema writer, + final Deque<String> location) { + location.addFirst(referenceToken); LOG.debug("Checking compatibility of reader {} with writer {}", reader, writer); final ReaderWriter pair = new ReaderWriter(reader, writer); - final SchemaCompatibilityResult existing = mMemoizeMap.get(pair); - if (existing != null) { - if (existing.getCompatibility() == SchemaCompatibilityType.RECURSION_IN_PROGRESS) { + SchemaCompatibilityResult result = mMemoizeMap.get(pair); + if (result != null) { + if (result.getCompatibility() == SchemaCompatibilityType.RECURSION_IN_PROGRESS) { // Break the recursion here. // schemas are compatible unless proven incompatible: - return SchemaCompatibilityResult.compatible(); + result = SchemaCompatibilityResult.compatible(); } - return existing; + } else { + // Mark this reader/writer pair as "in progress": + mMemoizeMap.put(pair, SchemaCompatibilityResult.recursionInProgress()); + result = calculateCompatibility(reader, writer, location); + mMemoizeMap.put(pair, result); } - // Mark this reader/writer pair as "in progress": - mMemoizeMap.put(pair, SchemaCompatibilityResult.recursionInProgress()); - final SchemaCompatibilityResult calculated = calculateCompatibility(reader, writer); - mMemoizeMap.put(pair, calculated); - return calculated; + location.removeFirst(); + return result; } /** @@ -251,14 +259,17 @@ public class SchemaCompatibility { * * @param reader Reader schema to test. * @param writer Writer schema to test. + * @param location Stack with which to track the location within the schema. * @return the compatibility of the reader/writer schema pair. */ private SchemaCompatibilityResult calculateCompatibility( final Schema reader, - final Schema writer + final Schema writer, + final Deque<String> location ) { assert (reader != null); assert (writer != null); + SchemaCompatibilityResult result = SchemaCompatibilityResult.compatible(); if (reader.getType() == writer.getType()) { switch (reader.getType()) { @@ -270,48 +281,44 @@ public class SchemaCompatibility { case DOUBLE: case BYTES: case STRING: { - return SchemaCompatibilityResult.compatible(); + return result; } case ARRAY: { - return getCompatibility(reader.getElementType(), writer.getElementType()); + return result.mergedWith(getCompatibility("items", reader.getElementType(), writer.getElementType(), location)); } case MAP: { - return getCompatibility(reader.getValueType(), writer.getValueType()); + return result.mergedWith(getCompatibility("values", reader.getValueType(), writer.getValueType(), location)); } case FIXED: { - SchemaCompatibilityResult nameCheck = checkSchemaNames(reader, writer); - if (nameCheck.getCompatibility() == SchemaCompatibilityType.INCOMPATIBLE) { - return nameCheck; - } - return checkFixedSize(reader, writer); + result = result.mergedWith(checkSchemaNames(reader, writer, location)); + return result.mergedWith(checkFixedSize(reader, writer, location)); } case ENUM: { - SchemaCompatibilityResult nameCheck = checkSchemaNames(reader, writer); - if (nameCheck.getCompatibility() == SchemaCompatibilityType.INCOMPATIBLE) { - return nameCheck; - } - return checkReaderEnumContainsAllWriterEnumSymbols(reader, writer); + result = result.mergedWith(checkSchemaNames(reader, writer, location)); + return result.mergedWith(checkReaderEnumContainsAllWriterEnumSymbols(reader, writer, location)); } case RECORD: { - SchemaCompatibilityResult nameCheck = checkSchemaNames(reader, writer); - if (nameCheck.getCompatibility() == SchemaCompatibilityType.INCOMPATIBLE) { - return nameCheck; - } - return checkReaderWriterRecordFields(reader, writer); + result = result.mergedWith(checkSchemaNames(reader, writer, location)); + return result.mergedWith(checkReaderWriterRecordFields(reader, writer, location)); } case UNION: { // Check that each individual branch of the writer union can be decoded: + int i = 0; for (final Schema writerBranch : writer.getTypes()) { + location.addFirst(Integer.toString(i)); SchemaCompatibilityResult compatibility = getCompatibility(reader, writerBranch); if (compatibility.getCompatibility() == SchemaCompatibilityType.INCOMPATIBLE) { - String msg = String.format("reader union lacking writer type: %s", + String message = String.format("reader union lacking writer type: %s", writerBranch.getType()); - return SchemaCompatibilityResult.incompatible( - SchemaIncompatibilityType.MISSING_UNION_BRANCH, reader, writer, msg); + result = result.mergedWith(SchemaCompatibilityResult.incompatible( + SchemaIncompatibilityType.MISSING_UNION_BRANCH, + reader, writer, message, asList(location))); } + location.removeFirst(); + i++; } // Each schema in the writer union can be decoded with the reader: - return SchemaCompatibilityResult.compatible(); + return result; } default: { @@ -325,65 +332,70 @@ public class SchemaCompatibility { // Reader compatible with all branches of a writer union is compatible if (writer.getType() == Schema.Type.UNION) { for (Schema s : writer.getTypes()) { - SchemaCompatibilityResult compat = getCompatibility(reader, s); - if (compat.getCompatibility() == SchemaCompatibilityType.INCOMPATIBLE) { - return compat; - } + result = result.mergedWith(getCompatibility(reader, s)); } - return SchemaCompatibilityResult.compatible(); + return result; } switch (reader.getType()) { case NULL: - return typeMismatch(reader, writer); + return result.mergedWith(typeMismatch(reader, writer, location)); case BOOLEAN: - return typeMismatch(reader, writer); + return result.mergedWith(typeMismatch(reader, writer, location)); case INT: - return typeMismatch(reader, writer); + return result.mergedWith(typeMismatch(reader, writer, location)); case LONG: { - return (writer.getType() == Type.INT) ? SchemaCompatibilityResult.compatible() - : typeMismatch(reader, writer); + return (writer.getType() == Type.INT) + ? result + : result.mergedWith(typeMismatch(reader, writer, location)); } case FLOAT: { - return ((writer.getType() == Type.INT) || (writer.getType() == Type.LONG)) - ? SchemaCompatibilityResult.compatible() : typeMismatch(reader, writer); + return ((writer.getType() == Type.INT) + || (writer.getType() == Type.LONG)) + ? result + : result.mergedWith(typeMismatch(reader, writer, location)); } case DOUBLE: { - return ((writer.getType() == Type.INT) || (writer.getType() == Type.LONG) - || (writer.getType() == Type.FLOAT)) ? SchemaCompatibilityResult.compatible() - : typeMismatch(reader, writer); + return ((writer.getType() == Type.INT) + || (writer.getType() == Type.LONG) + || (writer.getType() == Type.FLOAT)) + ? result + : result.mergedWith(typeMismatch(reader, writer, location)); } case BYTES: { - return (writer.getType() == Type.STRING) ? SchemaCompatibilityResult.compatible() - : typeMismatch(reader, writer); + return (writer.getType() == Type.STRING) + ? result + : result.mergedWith(typeMismatch(reader, writer, location)); } case STRING: { - return (writer.getType() == Type.BYTES) ? SchemaCompatibilityResult.compatible() - : typeMismatch(reader, writer); + return (writer.getType() == Type.BYTES) + ? result + : result.mergedWith(typeMismatch(reader, writer, location)); } case ARRAY: - return typeMismatch(reader, writer); + return result.mergedWith(typeMismatch(reader, writer, location)); case MAP: - return typeMismatch(reader, writer); + return result.mergedWith(typeMismatch(reader, writer, location)); case FIXED: - return typeMismatch(reader, writer); + return result.mergedWith(typeMismatch(reader, writer, location)); case ENUM: - return typeMismatch(reader, writer); + return result.mergedWith(typeMismatch(reader, writer, location)); case RECORD: - return typeMismatch(reader, writer); + return result.mergedWith(typeMismatch(reader, writer, location)); case UNION: { for (final Schema readerBranch : reader.getTypes()) { SchemaCompatibilityResult compatibility = getCompatibility(readerBranch, writer); if (compatibility.getCompatibility() == SchemaCompatibilityType.COMPATIBLE) { - return SchemaCompatibilityResult.compatible(); + return result; } } // No branch in the reader union has been found compatible with the writer schema: - String msg = String.format("reader union lacking writer type: %s", writer.getType()); - return SchemaCompatibilityResult - .incompatible(SchemaIncompatibilityType.MISSING_UNION_BRANCH, reader, writer, msg); + String message = String.format("reader union lacking writer type: %s", writer.getType()); + return result.mergedWith(SchemaCompatibilityResult.incompatible( + SchemaIncompatibilityType.MISSING_UNION_BRANCH, + reader, writer, message, asList(location))); } default: { @@ -394,65 +406,86 @@ public class SchemaCompatibility { } private SchemaCompatibilityResult checkReaderWriterRecordFields(final Schema reader, - final Schema writer) { + final Schema writer, + final Deque<String> location) { + SchemaCompatibilityResult result = SchemaCompatibilityResult.compatible(); + location.addFirst("fields"); // Check that each field in the reader record can be populated from the writer record: for (final Field readerField : reader.getFields()) { + location.addFirst(Integer.toString(readerField.pos())); final Field writerField = lookupWriterField(writer, readerField); if (writerField == null) { // Reader field does not correspond to any field in the writer record schema, so the // reader field must have a default value. if (readerField.defaultValue() == null) { // reader field has no default value - return SchemaCompatibilityResult.incompatible( + result = result.mergedWith(SchemaCompatibilityResult.incompatible( SchemaIncompatibilityType.READER_FIELD_MISSING_DEFAULT_VALUE, reader, writer, - readerField.name()); + readerField.name(), asList(location))); } } else { - SchemaCompatibilityResult compatibility = getCompatibility(readerField.schema(), - writerField.schema()); - if (compatibility.getCompatibility() == SchemaCompatibilityType.INCOMPATIBLE) { - return compatibility; - } + result = result.mergedWith(getCompatibility("type", readerField.schema(), + writerField.schema(), location)); } + // POP field index + location.removeFirst(); } // All fields in the reader record can be populated from the writer record: - return SchemaCompatibilityResult.compatible(); + // POP "fields" literal + location.removeFirst(); + return result; } private SchemaCompatibilityResult checkReaderEnumContainsAllWriterEnumSymbols( - final Schema reader, final Schema writer) { - final Set<String> symbols = new TreeSet<>(writer.getEnumSymbols()); + final Schema reader, final Schema writer, final Deque<String> location) { + SchemaCompatibilityResult result = SchemaCompatibilityResult.compatible(); + location.addFirst("symbols"); + final Set<String> symbols = new TreeSet<String>(writer.getEnumSymbols()); symbols.removeAll(reader.getEnumSymbols()); - return symbols.isEmpty() ? SchemaCompatibilityResult.compatible() - : SchemaCompatibilityResult.incompatible(SchemaIncompatibilityType.MISSING_ENUM_SYMBOLS, - reader, writer, symbols.toString()); + if (!symbols.isEmpty()) { + result = SchemaCompatibilityResult.incompatible( + SchemaIncompatibilityType.MISSING_ENUM_SYMBOLS, reader, writer, + symbols.toString(), asList(location)); + } + // POP "symbols" literal + location.removeFirst(); + return result; } - private SchemaCompatibilityResult checkFixedSize(final Schema reader, final Schema writer) { + private SchemaCompatibilityResult checkFixedSize(final Schema reader, final Schema writer, final Deque<String> location) { + SchemaCompatibilityResult result = SchemaCompatibilityResult.compatible(); + location.addFirst("size"); int actual = reader.getFixedSize(); int expected = writer.getFixedSize(); if (actual != expected) { - String msg = String.format("expected: %d, found: %d", expected, actual); - return SchemaCompatibilityResult.incompatible(SchemaIncompatibilityType.FIXED_SIZE_MISMATCH, - reader, writer, msg); + String message = String.format("expected: %d, found: %d", expected, actual); + result = SchemaCompatibilityResult.incompatible(SchemaIncompatibilityType.FIXED_SIZE_MISMATCH, + reader, writer, message, asList(location)); } - return SchemaCompatibilityResult.compatible(); + // POP "size" literal + location.removeFirst(); + return result; } - private SchemaCompatibilityResult checkSchemaNames(final Schema reader, final Schema writer) { + private SchemaCompatibilityResult checkSchemaNames(final Schema reader, final Schema writer, final Deque<String> location) { + SchemaCompatibilityResult result = SchemaCompatibilityResult.compatible(); + location.addFirst("name"); if (!schemaNameEquals(reader, writer)) { - String msg = String.format("expected: %s", writer.getFullName()); - return SchemaCompatibilityResult.incompatible(SchemaIncompatibilityType.NAME_MISMATCH, - reader, writer, msg); + String message = String.format("expected: %s", writer.getFullName()); + result = SchemaCompatibilityResult.incompatible( + SchemaIncompatibilityType.NAME_MISMATCH, + reader, writer, message, asList(location)); } - return SchemaCompatibilityResult.compatible(); + // POP "name" literal + location.removeFirst(); + return result; } - private SchemaCompatibilityResult typeMismatch(final Schema reader, final Schema writer) { - String msg = String.format("reader type: %s not compatible with writer type: %s", + private SchemaCompatibilityResult typeMismatch(final Schema reader, final Schema writer, final Deque<String> location) { + String message = String.format("reader type: %s not compatible with writer type: %s", reader.getType(), writer.getType()); return SchemaCompatibilityResult.incompatible(SchemaIncompatibilityType.TYPE_MISMATCH, reader, - writer, msg); + writer, message, asList(location)); } } @@ -480,26 +513,37 @@ public class SchemaCompatibility { * Immutable class representing details about a particular schema pair compatibility check. */ public static final class SchemaCompatibilityResult { - private final SchemaCompatibilityType mCompatibility; + + /** + * Merges the current {@code SchemaCompatibilityResult} with the supplied result into a new instance, combining the + * list of {@code Incompatibility Incompatibilities} and regressing to the + * {@code SchemaCompatibilityType#INCOMPATIBLE INCOMPATIBLE} state if any incompatibilities are encountered. + * + * @param toMerge The {@code SchemaCompatibilityResult} to merge with the current instance. + * @return A {@code SchemaCompatibilityResult} that combines the state of the current and supplied instances. + */ + public SchemaCompatibilityResult mergedWith(SchemaCompatibilityResult toMerge) { + List<Incompatibility> mergedIncompatibilities = new ArrayList<Incompatibility>(mIncompatibilities); + mergedIncompatibilities.addAll(toMerge.getIncompatibilities()); + SchemaCompatibilityType compatibilityType = mCompatibilityType == SchemaCompatibilityType.COMPATIBLE + ? toMerge.mCompatibilityType + : SchemaCompatibilityType.INCOMPATIBLE; + return new SchemaCompatibilityResult(compatibilityType, mergedIncompatibilities); + } + + private final SchemaCompatibilityType mCompatibilityType; // the below fields are only valid if INCOMPATIBLE - private final SchemaIncompatibilityType mSchemaIncompatibilityType; - private final Schema mReaderSubset; - private final Schema mWriterSubset; - private final String mMessage; + private final List<Incompatibility> mIncompatibilities; // cached objects for stateless details private static final SchemaCompatibilityResult COMPATIBLE = new SchemaCompatibilityResult( - SchemaCompatibilityType.COMPATIBLE, null, null, null, null); + SchemaCompatibilityType.COMPATIBLE, Collections.<Incompatibility> emptyList()); private static final SchemaCompatibilityResult RECURSION_IN_PROGRESS = new SchemaCompatibilityResult( - SchemaCompatibilityType.RECURSION_IN_PROGRESS, null, null, null, null); + SchemaCompatibilityType.RECURSION_IN_PROGRESS, Collections.<Incompatibility> emptyList()); - private SchemaCompatibilityResult(SchemaCompatibilityType type, - SchemaIncompatibilityType errorDetails, Schema readerDetails, Schema writerDetails, - String details) { - this.mCompatibility = type; - this.mSchemaIncompatibilityType = errorDetails; - this.mReaderSubset = readerDetails; - this.mWriterSubset = writerDetails; - this.mMessage = details; + private SchemaCompatibilityResult(SchemaCompatibilityType compatibilityType, + List<Incompatibility> incompatibilities) { + this.mCompatibilityType = compatibilityType; + this.mIncompatibilities = incompatibilities; } /** @@ -525,10 +569,15 @@ public class SchemaCompatibility { * @return a SchemaCompatibilityDetails object with INCOMPATIBLE SchemaCompatibilityType, and * state representing the violating part. */ - public static SchemaCompatibilityResult incompatible(SchemaIncompatibilityType error, - Schema reader, Schema writer, String details) { - return new SchemaCompatibilityResult(SchemaCompatibilityType.INCOMPATIBLE, error, reader, - writer, details); + public static SchemaCompatibilityResult incompatible( + SchemaIncompatibilityType incompatibilityType, + Schema readerFragment, + Schema writerFragment, + String message, + List<String> location + ) { + Incompatibility incompatibility = new Incompatibility(incompatibilityType, readerFragment, writerFragment, message, location); + return new SchemaCompatibilityResult(SchemaCompatibilityType.INCOMPATIBLE, Collections.singletonList(incompatibility)); } /** @@ -536,57 +585,146 @@ public class SchemaCompatibility { * @return a SchemaCompatibilityType instance, always non-null */ public SchemaCompatibilityType getCompatibility() { - return mCompatibility; + return mCompatibilityType; } /** - * If the compatibility is INCOMPATIBLE, returns the SchemaIncompatibilityType (first thing that - * was incompatible), otherwise null. - * @return a SchemaIncompatibilityType instance, or null + * If the compatibility is INCOMPATIBLE, returns {@link Incompatibility Incompatibilities} found, otherwise an empty + * list. + * @return a list of {@link Incompatibility Incompatibilities}, may be empty, never null. */ - public SchemaIncompatibilityType getIncompatibility() { - return mSchemaIncompatibilityType; + public List<Incompatibility> getIncompatibilities() { + return mIncompatibilities; + } + + /** {@inheritDoc} */ + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((mCompatibilityType == null) ? 0 : mCompatibilityType.hashCode()); + result = prime * result + + ((mIncompatibilities == null) ? 0 : mIncompatibilities.hashCode()); + return result; + } + + /** {@inheritDoc} */ + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + SchemaCompatibilityResult other = (SchemaCompatibilityResult) obj; + if (mIncompatibilities == null) { + if (other.mIncompatibilities != null) + return false; + } else if (!mIncompatibilities.equals(other.mIncompatibilities)) + return false; + if (mCompatibilityType != other.mCompatibilityType) + return false; + return true; + } + + /** {@inheritDoc} */ + @Override + public String toString() { + return String.format( + "SchemaCompatibilityResult{compatibility:%s, incompatibilities:%s}", + mCompatibilityType, mIncompatibilities); + } + } + // ----------------------------------------------------------------------------------------------- + + public static final class Incompatibility { + private final SchemaIncompatibilityType mType; + private final Schema mReaderFragment; + private final Schema mWriterFragment; + private final String mMessage; + private final List<String> mLocation; + + Incompatibility( + SchemaIncompatibilityType type, + Schema readerFragment, + Schema writerFragment, + String message, + List<String> location) { + super(); + this.mType = type; + this.mReaderFragment = readerFragment; + this.mWriterFragment = writerFragment; + this.mMessage = message; + this.mLocation = location; } /** - * If the compatibility is INCOMPATIBLE, returns the first part of the reader schema that failed - * compatibility check. - * @return a Schema instance (part of the reader schema), or null + * Returns the SchemaIncompatibilityType. + * @return a SchemaIncompatibilityType instance. */ - public Schema getReaderSubset() { - return mReaderSubset; + public SchemaIncompatibilityType getType() { + return mType; } /** - * If the compatibility is INCOMPATIBLE, returns the first part of the writer schema that failed - * compatibility check. - * @return a Schema instance (part of the writer schema), or null + * Returns the fragment of the reader schema that failed compatibility check. + * @return a Schema instance (fragment of the reader schema). */ - public Schema getWriterSubset() { - return mWriterSubset; + public Schema getReaderFragment() { + return mReaderFragment; } /** - * If the compatibility is INCOMPATIBLE, returns a human-readable string with more details about - * what failed. Syntax depends on the SchemaIncompatibilityType. - * @see #getIncompatibility() - * @return a String with details about the incompatibility, or null + * Returns the fragment of the writer schema that failed compatibility check. + * @return a Schema instance (fragment of the writer schema). + */ + public Schema getWriterFragment() { + return mWriterFragment; + } + + /** + * Returns a human-readable message with more details about what failed. Syntax depends on the + * SchemaIncompatibilityType. + * @see #getType() + * @return a String with details about the incompatibility. */ public String getMessage() { return mMessage; } + /** + * Returns a <a href="https://tools.ietf.org/html/draft-ietf-appsawg-json-pointer-08";>JSON Pointer</a> describing + * the node location within the schema's JSON document tree where the incompatibility was encountered. + * @return JSON Pointer encoded as a string. + */ + public String getLocation() { + StringBuilder s = new StringBuilder("/"); + boolean first = true; + // ignore root element + for (String coordinate : mLocation.subList(1, mLocation.size())) { + if (first) { + first = false; + } else { + s.append('/'); + } + // Apply JSON pointer escaping. + s.append(coordinate.replace("~", "~0").replace("/", "~1")); + } + return s.toString(); + } + /** {@inheritDoc} */ @Override public int hashCode() { final int prime = 31; int result = 1; + result = prime * result + ((mType == null) ? 0 : mType.hashCode()); + result = prime * result + ((mReaderFragment == null) ? 0 : mReaderFragment.hashCode()); + result = prime * result + ((mWriterFragment == null) ? 0 : mWriterFragment.hashCode()); result = prime * result + ((mMessage == null) ? 0 : mMessage.hashCode()); - result = prime * result + ((mReaderSubset == null) ? 0 : mReaderSubset.hashCode()); - result = prime * result + ((mCompatibility == null) ? 0 : mCompatibility.hashCode()); - result = prime * result - + ((mSchemaIncompatibilityType == null) ? 0 : mSchemaIncompatibilityType.hashCode()); - result = prime * result + ((mWriterSubset == null) ? 0 : mWriterSubset.hashCode()); + result = prime * result + ((mLocation == null) ? 0 : mLocation.hashCode()); return result; } @@ -602,32 +740,36 @@ public class SchemaCompatibility { if (getClass() != obj.getClass()) { return false; } - SchemaCompatibilityResult other = (SchemaCompatibilityResult) obj; - if (mMessage == null) { - if (other.mMessage != null) { - return false; - } - } else if (!mMessage.equals(other.mMessage)) { + Incompatibility other = (Incompatibility) obj; + if (mType != other.mType) { return false; } - if (mReaderSubset == null) { - if (other.mReaderSubset != null) { + if (mReaderFragment == null) { + if (other.mReaderFragment != null) { return false; } - } else if (!mReaderSubset.equals(other.mReaderSubset)) { + } else if (!mReaderFragment.equals(other.mReaderFragment)) { return false; } - if (mCompatibility != other.mCompatibility) { + if (mWriterFragment == null) { + if (other.mWriterFragment != null) { + return false; + } + } else if (!mWriterFragment.equals(other.mWriterFragment)) { return false; } - if (mSchemaIncompatibilityType != other.mSchemaIncompatibilityType) { + if (mMessage == null) { + if (other.mMessage != null) { + return false; + } + } else if (!mMessage.equals(other.mMessage)) { return false; } - if (mWriterSubset == null) { - if (other.mWriterSubset != null) { + if (mLocation == null) { + if (other.mLocation != null) { return false; } - } else if (!mWriterSubset.equals(other.mWriterSubset)) { + } else if (!mLocation.equals(other.mLocation)) { return false; } return true; @@ -637,8 +779,8 @@ public class SchemaCompatibility { @Override public String toString() { return String.format( - "SchemaCompatibilityDetails{compatibility:%s, type:%s, readerSubset:%s, writerSubset:%s, message:%s}", - mCompatibility, mSchemaIncompatibilityType, mReaderSubset, mWriterSubset, mMessage); + "Incompatibility{type:%s, location:%s, message:%s, reader:%s, writer:%s}", + mType, getLocation(), mMessage, mReaderFragment, mWriterFragment); } } // ----------------------------------------------------------------------------------------------- @@ -663,7 +805,7 @@ public class SchemaCompatibility { /** * Constructs a new instance. - * @param result of the schema compatibility. + * @param result The result of the compatibility check. * @param reader schema that was validated. * @param writer schema that was validated. * @param description of this compatibility result. @@ -755,4 +897,10 @@ public class SchemaCompatibility { private static boolean objectsEqual(Object obj1, Object obj2) { return (obj1 == obj2) || ((obj1 != null) && obj1.equals(obj2)); } + + private static List<String> asList(Deque<String> deque) { + List<String> list = new ArrayList<String>(deque); + Collections.reverse(list); + return Collections.unmodifiableList(list); + } } http://git-wip-us.apache.org/repos/asf/avro/blob/2df0775d/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibility.java ---------------------------------------------------------------------- diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibility.java b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibility.java index 6b38cbd..591fe94 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibility.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibility.java @@ -17,6 +17,7 @@ */ package org.apache.avro; +import static java.util.Arrays.asList; import static org.apache.avro.SchemaCompatibility.checkReaderWriterCompatibility; import static org.apache.avro.TestSchemas.A_DINT_B_DINT_RECORD1; import static org.apache.avro.TestSchemas.A_DINT_RECORD1; @@ -61,8 +62,13 @@ import static org.apache.avro.TestSchemas.list; import static org.junit.Assert.assertEquals; import java.io.ByteArrayOutputStream; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Collections; +import java.util.Deque; import java.util.List; +import org.apache.avro.SchemaCompatibility.Incompatibility; import org.apache.avro.SchemaCompatibility.SchemaCompatibilityResult; import org.apache.avro.SchemaCompatibility.SchemaCompatibilityType; import org.apache.avro.SchemaCompatibility.SchemaIncompatibilityType; @@ -163,12 +169,13 @@ public class TestSchemaCompatibility { new Schema.Field("oldfield1", INT_SCHEMA, null, null), new Schema.Field("newfield1", INT_SCHEMA, null, null)); final Schema reader = Schema.createRecord(readerFields); - // Test new field without default value. SchemaPairCompatibility compatibility = checkReaderWriterCompatibility(reader, WRITER_SCHEMA); + + // Test new field without default value. assertEquals(SchemaCompatibility.SchemaCompatibilityType.INCOMPATIBLE, compatibility.getType()); assertEquals(SchemaCompatibility.SchemaCompatibilityResult.incompatible( SchemaIncompatibilityType.READER_FIELD_MISSING_DEFAULT_VALUE, reader, WRITER_SCHEMA, - "newfield1"), compatibility.getResult()); + "newfield1", asList("", "fields", "1")), compatibility.getResult()); assertEquals( String.format( "Data encoded using writer schema:%n%s%n" @@ -195,7 +202,8 @@ public class TestSchemaCompatibility { SchemaIncompatibilityType.TYPE_MISMATCH, invalidReader, STRING_ARRAY_SCHEMA, - "reader type: MAP not compatible with writer type: ARRAY"), + "reader type: MAP not compatible with writer type: ARRAY", + asList("")), invalidReader, STRING_ARRAY_SCHEMA, String.format( @@ -227,7 +235,8 @@ public class TestSchemaCompatibility { SchemaIncompatibilityType.TYPE_MISMATCH, INT_SCHEMA, STRING_SCHEMA, - "reader type: INT not compatible with writer type: STRING"), + "reader type: INT not compatible with writer type: STRING", + asList("")), INT_SCHEMA, STRING_SCHEMA, String.format( @@ -247,8 +256,8 @@ public class TestSchemaCompatibility { /** Reader union schema must contain all writer union branches. */ @Test public void testUnionReaderWriterSubsetIncompatibility() { - final Schema unionWriter = Schema.createUnion(list(INT_SCHEMA, STRING_SCHEMA)); - final Schema unionReader = Schema.createUnion(list(STRING_SCHEMA)); + final Schema unionWriter = Schema.createUnion(list(INT_SCHEMA, STRING_SCHEMA, LONG_SCHEMA)); + final Schema unionReader = Schema.createUnion(list(INT_SCHEMA, STRING_SCHEMA)); final SchemaPairCompatibility result = checkReaderWriterCompatibility(unionReader, unionWriter); assertEquals(SchemaCompatibilityType.INCOMPATIBLE, result.getType()); @@ -299,10 +308,10 @@ public class TestSchemaCompatibility { new ReaderWriter(INT_UNION_SCHEMA, INT_UNION_SCHEMA), new ReaderWriter(INT_STRING_UNION_SCHEMA, STRING_INT_UNION_SCHEMA), new ReaderWriter(INT_UNION_SCHEMA, EMPTY_UNION_SCHEMA), + new ReaderWriter(LONG_UNION_SCHEMA, EMPTY_UNION_SCHEMA), new ReaderWriter(LONG_UNION_SCHEMA, INT_UNION_SCHEMA), new ReaderWriter(FLOAT_UNION_SCHEMA, INT_UNION_SCHEMA), new ReaderWriter(DOUBLE_UNION_SCHEMA, INT_UNION_SCHEMA), - new ReaderWriter(LONG_UNION_SCHEMA, EMPTY_UNION_SCHEMA), new ReaderWriter(FLOAT_UNION_SCHEMA, LONG_UNION_SCHEMA), new ReaderWriter(DOUBLE_UNION_SCHEMA, LONG_UNION_SCHEMA), new ReaderWriter(FLOAT_UNION_SCHEMA, EMPTY_UNION_SCHEMA), @@ -359,18 +368,42 @@ public class TestSchemaCompatibility { * per error case (for easier pinpointing of errors). The method to validate incompatibility is * still here. */ + public static void validateIncompatibleSchemas( + Schema reader, + Schema writer, + SchemaIncompatibilityType incompatibility, + String message, + String location + ) { + validateIncompatibleSchemas( + reader, + writer, + asList(incompatibility), + asList(message), + asList(location) + ); + } + + // ----------------------------------------------------------------------------------------------- + public static void validateIncompatibleSchemas(Schema reader, Schema writer, - SchemaIncompatibilityType incompatibility, String details) { + List<SchemaIncompatibilityType> incompatibilityTypes, List<String> messages, List<String> locations) { SchemaPairCompatibility compatibility = checkReaderWriterCompatibility(reader, writer); - SchemaCompatibilityResult compatibilityDetails = compatibility.getResult(); - assertEquals(incompatibility, compatibilityDetails.getIncompatibility()); - Schema readerSubset = compatibilityDetails.getReaderSubset(); - Schema writerSubset = compatibilityDetails.getWriterSubset(); - assertSchemaContains(readerSubset, reader); - assertSchemaContains(writerSubset, writer); + SchemaCompatibilityResult compatibilityResult = compatibility.getResult(); assertEquals(reader, compatibility.getReader()); assertEquals(writer, compatibility.getWriter()); - assertEquals(details, compatibilityDetails.getMessage()); + assertEquals(SchemaCompatibilityType.INCOMPATIBLE, compatibilityResult.getCompatibility()); + + assertEquals(incompatibilityTypes.size(), compatibilityResult.getIncompatibilities().size()); + for (int i = 0 ; i < incompatibilityTypes.size(); i++) { + Incompatibility incompatibility = compatibilityResult.getIncompatibilities().get(i); + assertSchemaContains(incompatibility.getReaderFragment(), reader); + assertSchemaContains(incompatibility.getWriterFragment(), writer); + assertEquals(incompatibilityTypes.get(i), incompatibility.getType()); + assertEquals(messages.get(i), incompatibility.getMessage()); + assertEquals(locations.get(i), incompatibility.getLocation()); + } + String description = String.format( "Data encoded using writer schema:%n%s%n" + "will or may fail to decode using reader schema:%n%s%n", @@ -515,4 +548,12 @@ public class TestSchemaCompatibility { expectedDecodedDatum, decodedDatum); } } + + Deque<String> asDeqeue(String... args) { + Deque<String> dq = new ArrayDeque<String>(); + List<String> x = Arrays.asList(args); + Collections.reverse(x); + dq.addAll(x); + return dq; + } } http://git-wip-us.apache.org/repos/asf/avro/blob/2df0775d/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityFixedSizeMismatch.java ---------------------------------------------------------------------- diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityFixedSizeMismatch.java b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityFixedSizeMismatch.java index 7403856..cc201ab 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityFixedSizeMismatch.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityFixedSizeMismatch.java @@ -18,6 +18,8 @@ package org.apache.avro; import static org.apache.avro.TestSchemaCompatibility.validateIncompatibleSchemas; +import static org.apache.avro.TestSchemas.A_DINT_B_DFIXED_4_BYTES_RECORD1; +import static org.apache.avro.TestSchemas.A_DINT_B_DFIXED_8_BYTES_RECORD1; import static org.apache.avro.TestSchemas.FIXED_4_BYTES; import static org.apache.avro.TestSchemas.FIXED_8_BYTES; import java.util.ArrayList; @@ -35,9 +37,11 @@ public class TestSchemaCompatibilityFixedSizeMismatch { @Parameters(name = "r: {0} | w: {1}") public static Iterable<Object[]> data() { Object[][] fields = { // - { FIXED_4_BYTES, FIXED_8_BYTES, "expected: 8, found: 4" }, - { FIXED_8_BYTES, FIXED_4_BYTES, "expected: 4, found: 8" } }; - List<Object[]> list = new ArrayList<>(fields.length); + { FIXED_4_BYTES, FIXED_8_BYTES, "expected: 8, found: 4", "/size" }, + { FIXED_8_BYTES, FIXED_4_BYTES, "expected: 4, found: 8", "/size" }, + { A_DINT_B_DFIXED_8_BYTES_RECORD1, A_DINT_B_DFIXED_4_BYTES_RECORD1, "expected: 4, found: 8", "/fields/1/type/size" }, + { A_DINT_B_DFIXED_4_BYTES_RECORD1, A_DINT_B_DFIXED_8_BYTES_RECORD1, "expected: 8, found: 4", "/fields/1/type/size" }, }; + List<Object[]> list = new ArrayList<Object[]>(fields.length); for (Object[] schemas : fields) { list.add(schemas); } @@ -50,10 +54,12 @@ public class TestSchemaCompatibilityFixedSizeMismatch { public Schema writer; @Parameter(2) public String details; + @Parameter(3) + public String location; @Test public void testFixedSizeMismatchSchemas() throws Exception { validateIncompatibleSchemas(reader, writer, SchemaIncompatibilityType.FIXED_SIZE_MISMATCH, - details); + details, location); } } http://git-wip-us.apache.org/repos/asf/avro/blob/2df0775d/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityMissingEnumSymbols.java ---------------------------------------------------------------------- diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityMissingEnumSymbols.java b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityMissingEnumSymbols.java index d457283..893c4a6 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityMissingEnumSymbols.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityMissingEnumSymbols.java @@ -43,9 +43,9 @@ public class TestSchemaCompatibilityMissingEnumSymbols { @Parameters(name = "r: {0} | w: {1}") public static Iterable<Object[]> data() { Object[][] fields = { // - { ENUM1_AB_SCHEMA, ENUM1_ABC_SCHEMA, "[C]" }, { ENUM1_BC_SCHEMA, ENUM1_ABC_SCHEMA, "[A]" }, - { RECORD1_WITH_ENUM_AB, RECORD1_WITH_ENUM_ABC, "[C]" } }; - List<Object[]> list = new ArrayList<>(fields.length); + { ENUM1_AB_SCHEMA, ENUM1_ABC_SCHEMA, "[C]", "/symbols" }, { ENUM1_BC_SCHEMA, ENUM1_ABC_SCHEMA, "[A]", "/symbols" }, + { RECORD1_WITH_ENUM_AB, RECORD1_WITH_ENUM_ABC, "[C]", "/fields/0/type/symbols" } }; + List<Object[]> list = new ArrayList<Object[]>(fields.length); for (Object[] schemas : fields) { list.add(schemas); } @@ -58,10 +58,12 @@ public class TestSchemaCompatibilityMissingEnumSymbols { public Schema writer; @Parameter(2) public String details; + @Parameter(3) + public String location; @Test public void testTypeMismatchSchemas() throws Exception { validateIncompatibleSchemas(reader, writer, SchemaIncompatibilityType.MISSING_ENUM_SYMBOLS, - details); + details, location); } } http://git-wip-us.apache.org/repos/asf/avro/blob/2df0775d/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityMissingUnionBranch.java ---------------------------------------------------------------------- diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityMissingUnionBranch.java b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityMissingUnionBranch.java index 1a31078..fb47ef3 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityMissingUnionBranch.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityMissingUnionBranch.java @@ -17,7 +17,10 @@ */ package org.apache.avro; +import static java.util.Arrays.asList; import static org.apache.avro.TestSchemaCompatibility.validateIncompatibleSchemas; +import static org.apache.avro.TestSchemas.A_DINT_B_DINT_STRING_UNION_RECORD1; +import static org.apache.avro.TestSchemas.A_DINT_B_DINT_UNION_RECORD1; import static org.apache.avro.TestSchemas.BOOLEAN_SCHEMA; import static org.apache.avro.TestSchemas.BYTES_UNION_SCHEMA; import static org.apache.avro.TestSchemas.DOUBLE_UNION_SCHEMA; @@ -35,6 +38,7 @@ import static org.apache.avro.TestSchemas.NULL_SCHEMA; import static org.apache.avro.TestSchemas.STRING_UNION_SCHEMA; import static org.apache.avro.TestSchemas.list; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.apache.avro.SchemaCompatibility.SchemaIncompatibilityType; import org.junit.Test; @@ -71,25 +75,28 @@ public class TestSchemaCompatibilityMissingUnionBranch { @Parameters(name = "r: {0} | w: {1}") public static Iterable<Object[]> data() { Object[][] fields = { // - { INT_UNION_SCHEMA, INT_STRING_UNION_SCHEMA, "reader union lacking writer type: STRING" }, - { STRING_UNION_SCHEMA, INT_STRING_UNION_SCHEMA, "reader union lacking writer type: INT" }, - { INT_UNION_SCHEMA, UNION_INT_RECORD1, "reader union lacking writer type: RECORD" }, - { INT_UNION_SCHEMA, UNION_INT_RECORD2, "reader union lacking writer type: RECORD" }, + { INT_UNION_SCHEMA, INT_STRING_UNION_SCHEMA, asList("reader union lacking writer type: STRING"), asList("/1") }, + { STRING_UNION_SCHEMA, INT_STRING_UNION_SCHEMA, asList("reader union lacking writer type: INT"), asList("/0") }, + { INT_UNION_SCHEMA, UNION_INT_RECORD1, asList("reader union lacking writer type: RECORD"), asList("/1") }, + { INT_UNION_SCHEMA, UNION_INT_RECORD2, asList("reader union lacking writer type: RECORD"), asList("/1") }, // more info in the subset schemas - { UNION_INT_RECORD1, UNION_INT_RECORD2, "reader union lacking writer type: RECORD" }, - { INT_UNION_SCHEMA, UNION_INT_ENUM1_AB, "reader union lacking writer type: ENUM" }, - { INT_UNION_SCHEMA, UNION_INT_FIXED_4_BYTES, "reader union lacking writer type: FIXED" }, - { INT_UNION_SCHEMA, UNION_INT_BOOLEAN, "reader union lacking writer type: BOOLEAN" }, - { INT_UNION_SCHEMA, LONG_UNION_SCHEMA, "reader union lacking writer type: LONG" }, - { INT_UNION_SCHEMA, FLOAT_UNION_SCHEMA, "reader union lacking writer type: FLOAT" }, - { INT_UNION_SCHEMA, DOUBLE_UNION_SCHEMA, "reader union lacking writer type: DOUBLE" }, - { INT_UNION_SCHEMA, BYTES_UNION_SCHEMA, "reader union lacking writer type: BYTES" }, - { INT_UNION_SCHEMA, UNION_INT_ARRAY_INT, "reader union lacking writer type: ARRAY" }, - { INT_UNION_SCHEMA, UNION_INT_MAP_INT, "reader union lacking writer type: MAP" }, - { INT_UNION_SCHEMA, UNION_INT_NULL, "reader union lacking writer type: NULL" }, + { UNION_INT_RECORD1, UNION_INT_RECORD2, asList("reader union lacking writer type: RECORD"), asList("/1") }, + { INT_UNION_SCHEMA, UNION_INT_ENUM1_AB, asList("reader union lacking writer type: ENUM"), asList("/1") }, + { INT_UNION_SCHEMA, UNION_INT_FIXED_4_BYTES, asList("reader union lacking writer type: FIXED"), asList("/1") }, + { INT_UNION_SCHEMA, UNION_INT_BOOLEAN, asList("reader union lacking writer type: BOOLEAN"), asList("/1") }, + { INT_UNION_SCHEMA, LONG_UNION_SCHEMA, asList("reader union lacking writer type: LONG"), asList("/0") }, + { INT_UNION_SCHEMA, FLOAT_UNION_SCHEMA, asList("reader union lacking writer type: FLOAT"), asList("/0") }, + { INT_UNION_SCHEMA, DOUBLE_UNION_SCHEMA, asList("reader union lacking writer type: DOUBLE"), asList("/0") }, + { INT_UNION_SCHEMA, BYTES_UNION_SCHEMA, asList("reader union lacking writer type: BYTES"), asList("/0") }, + { INT_UNION_SCHEMA, UNION_INT_ARRAY_INT, asList("reader union lacking writer type: ARRAY"), asList("/1") }, + { INT_UNION_SCHEMA, UNION_INT_MAP_INT, asList("reader union lacking writer type: MAP"), asList("/1") }, + { INT_UNION_SCHEMA, UNION_INT_NULL, asList("reader union lacking writer type: NULL"), asList("/1") }, { INT_UNION_SCHEMA, INT_LONG_FLOAT_DOUBLE_UNION_SCHEMA, - "reader union lacking writer type: LONG" }, }; - List<Object[]> list = new ArrayList<>(fields.length); + asList("reader union lacking writer type: LONG", "reader union lacking writer type: FLOAT", "reader union lacking writer type: DOUBLE"), + asList("/1", "/2", "/3") }, + { A_DINT_B_DINT_UNION_RECORD1, A_DINT_B_DINT_STRING_UNION_RECORD1, + asList("reader union lacking writer type: STRING"), asList("/fields/1/type/1") } }; + List<Object[]> list = new ArrayList<Object[]>(fields.length); for (Object[] schemas : fields) { list.add(schemas); } @@ -101,11 +108,13 @@ public class TestSchemaCompatibilityMissingUnionBranch { @Parameter(1) public Schema writer; @Parameter(2) - public String details; + public List<String> details; + @Parameter(3) + public List<String> location; @Test public void testMissingUnionBranch() throws Exception { - validateIncompatibleSchemas(reader, writer, SchemaIncompatibilityType.MISSING_UNION_BRANCH, - details); + List<SchemaIncompatibilityType> types = Collections.nCopies(details.size(), SchemaIncompatibilityType.MISSING_UNION_BRANCH); + validateIncompatibleSchemas(reader, writer, types, details, location); } } http://git-wip-us.apache.org/repos/asf/avro/blob/2df0775d/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityMultiple.java ---------------------------------------------------------------------- diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityMultiple.java b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityMultiple.java new file mode 100644 index 0000000..2d9b0e1 --- /dev/null +++ b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityMultiple.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.avro; + +import static org.apache.avro.TestSchemaCompatibility.validateIncompatibleSchemas; + +import java.util.Arrays; +import java.util.List; + +import org.apache.avro.SchemaCompatibility.SchemaIncompatibilityType; +import org.junit.Test; + +public class TestSchemaCompatibilityMultiple { + + @Test + public void testMultipleIncompatibilities() throws Exception { + Schema reader = SchemaBuilder.record("base").fields() + // 0 + .name("check_enum_symbols_field") + .type().enumeration("check_enum_symbols_type").symbols("A", "C").noDefault() + // 1 + .name("check_enum_name_field") + .type().enumeration("check_enum_name_type").symbols("A", "B", "C", "D").noDefault() + // 2 + .name("type_mismatch_field") + .type().stringType().noDefault() + // 3 + .name("sub_record") + .type().record("sub_record_type").fields() + // 3.0 + .name("identical_1_field") + .type().longType().longDefault(42L) + // 3.1 + .name("extra_no_default_field") + .type().longType().noDefault() + // 3.2 + .name("fixed_length_mismatch_field") + .type().fixed("fixed_length_mismatch_type").size(4).noDefault() + // 3.3 + .name("union_missing_branches_field") + .type().unionOf().booleanType().endUnion().noDefault() + // 3.4 + .name("reader_union_does_not_support_type_field") + .type().unionOf().booleanType().endUnion().noDefault() + // 3.5 + .name("record_fqn_mismatch_field") + .type().record("recordA").namespace("not_nsA").fields() + // 3.5.0 + .name("A_field_0") + .type().booleanType().booleanDefault(true) + // 3.5.1 + .name("array_type_mismatch_field") + .type().array().items().stringType().noDefault() + // EOR + .endRecord().noDefault() + // EOR + .endRecord().noDefault() + // EOR + .endRecord(); + + Schema writer = SchemaBuilder.record("base").fields() + // 0 + .name("check_enum_symbols_field") + .type().enumeration("check_enum_symbols_type").symbols("A", "B", "C", "D").noDefault() + // 1 + .name("check_enum_name_field") + .type().enumeration("check_enum_name_type_ERR").symbols("A", "B", "C", "D").noDefault() + // 2 + .name("type_mismatch_field") + .type().longType().noDefault() + // 3 + .name("sub_record") + .type().record("sub_record_type").fields() + // 3.0 + .name("identical_1_field") + .type().longType().longDefault(42L) + // 3.1 + // MISSING FIELD + // 3.2 + .name("fixed_length_mismatch_field") + .type().fixed("fixed_length_mismatch_type").size(8).noDefault() + // 3.3 + .name("union_missing_branches_field") + .type().unionOf().booleanType().and().doubleType().and().stringType().endUnion().noDefault() + // 3.4 + .name("reader_union_does_not_support_type_field") + .type().longType().noDefault() + // 3.5 + .name("record_fqn_mismatch_field") + .type().record("recordA").namespace("nsA").fields() + // 3.5.0 + .name("A_field_0") + .type().booleanType().booleanDefault(true) + // 3.5.1 + .name("array_type_mismatch_field") + .type().array().items().booleanType().noDefault() + // EOR + .endRecord().noDefault() + // EOR + .endRecord().noDefault() + // EOR + .endRecord(); + + List<SchemaIncompatibilityType> types = Arrays.asList( + SchemaIncompatibilityType.MISSING_ENUM_SYMBOLS, + SchemaIncompatibilityType.NAME_MISMATCH, + SchemaIncompatibilityType.TYPE_MISMATCH, + SchemaIncompatibilityType.READER_FIELD_MISSING_DEFAULT_VALUE, + SchemaIncompatibilityType.FIXED_SIZE_MISMATCH, + SchemaIncompatibilityType.MISSING_UNION_BRANCH, + SchemaIncompatibilityType.MISSING_UNION_BRANCH, + SchemaIncompatibilityType.MISSING_UNION_BRANCH, + SchemaIncompatibilityType.NAME_MISMATCH, + SchemaIncompatibilityType.TYPE_MISMATCH + ); + List<String> details = Arrays.asList( + "[B, D]", + "expected: check_enum_name_type_ERR", + "reader type: STRING not compatible with writer type: LONG", + "extra_no_default_field", + "expected: 8, found: 4", + "reader union lacking writer type: DOUBLE", + "reader union lacking writer type: STRING", + "reader union lacking writer type: LONG", + "expected: nsA.recordA", + "reader type: STRING not compatible with writer type: BOOLEAN" + ); + List<String> location = Arrays.asList( + "/fields/0/type/symbols", + "/fields/1/type/name", + "/fields/2/type", + "/fields/3/type/fields/1", + "/fields/3/type/fields/2/type/size", + "/fields/3/type/fields/3/type/1", + "/fields/3/type/fields/3/type/2", + "/fields/3/type/fields/4/type", + "/fields/3/type/fields/5/type/name", + "/fields/3/type/fields/5/type/fields/1/type/items" + ); + + validateIncompatibleSchemas(reader, writer, types, details, location); + } +} http://git-wip-us.apache.org/repos/asf/avro/blob/2df0775d/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityNameMismatch.java ---------------------------------------------------------------------- diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityNameMismatch.java b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityNameMismatch.java index 7d4bf6f..ebc2309 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityNameMismatch.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityNameMismatch.java @@ -18,6 +18,8 @@ package org.apache.avro; import static org.apache.avro.TestSchemaCompatibility.validateIncompatibleSchemas; +import static org.apache.avro.TestSchemas.A_DINT_B_DENUM_1_RECORD1; +import static org.apache.avro.TestSchemas.A_DINT_B_DENUM_2_RECORD1; import static org.apache.avro.TestSchemas.EMPTY_RECORD1; import static org.apache.avro.TestSchemas.EMPTY_RECORD2; import static org.apache.avro.TestSchemas.ENUM1_AB_SCHEMA; @@ -45,11 +47,12 @@ public class TestSchemaCompatibilityNameMismatch { @Parameters(name = "r: {0} | w: {1}") public static Iterable<Object[]> data() { Object[][] fields = { // - { ENUM1_AB_SCHEMA, ENUM2_AB_SCHEMA, "expected: Enum2" }, - { EMPTY_RECORD2, EMPTY_RECORD1, "expected: Record1" }, - { FIXED_4_BYTES, FIXED_4_ANOTHER_NAME, "expected: AnotherName" }, { FIXED_4_NAMESPACE_V1, - FIXED_4_NAMESPACE_V2, "expected: org.apache.avro.tests.v_2_0.Fixed" } }; - List<Object[]> list = new ArrayList<>(fields.length); + { ENUM1_AB_SCHEMA, ENUM2_AB_SCHEMA, "expected: Enum2", "/name" }, + { EMPTY_RECORD2, EMPTY_RECORD1, "expected: Record1", "/name" }, + { FIXED_4_BYTES, FIXED_4_ANOTHER_NAME, "expected: AnotherName", "/name" }, { FIXED_4_NAMESPACE_V1, + FIXED_4_NAMESPACE_V2, "expected: org.apache.avro.tests.v_2_0.Fixed", "/name" }, + { A_DINT_B_DENUM_1_RECORD1, A_DINT_B_DENUM_2_RECORD1, "expected: Enum2", "/fields/1/type/name" } }; + List<Object[]> list = new ArrayList<Object[]>(fields.length); for (Object[] schemas : fields) { list.add(schemas); } @@ -62,9 +65,11 @@ public class TestSchemaCompatibilityNameMismatch { public Schema writer; @Parameter(2) public String details; + @Parameter(3) + public String location; @Test public void testNameMismatchSchemas() throws Exception { - validateIncompatibleSchemas(reader, writer, SchemaIncompatibilityType.NAME_MISMATCH, details); + validateIncompatibleSchemas(reader, writer, SchemaIncompatibilityType.NAME_MISMATCH, details, location); } } http://git-wip-us.apache.org/repos/asf/avro/blob/2df0775d/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityReaderFieldMissingDefaultValue.java ---------------------------------------------------------------------- diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityReaderFieldMissingDefaultValue.java b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityReaderFieldMissingDefaultValue.java index 06a7dcb..b011bc9 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityReaderFieldMissingDefaultValue.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityReaderFieldMissingDefaultValue.java @@ -32,12 +32,11 @@ import org.junit.runners.Parameterized.Parameters; @RunWith(Parameterized.class) public class TestSchemaCompatibilityReaderFieldMissingDefaultValue { - @Parameters(name = "r: {0} | w: {1}") public static Iterable<Object[]> data() { Object[][] fields = { // - { A_INT_RECORD1, EMPTY_RECORD1, "a" }, { A_INT_B_DINT_RECORD1, EMPTY_RECORD1, "a" } }; - List<Object[]> list = new ArrayList<>(fields.length); + { A_INT_RECORD1, EMPTY_RECORD1, "a", "/fields/0" }, { A_INT_B_DINT_RECORD1, EMPTY_RECORD1, "a", "/fields/0" } }; + List<Object[]> list = new ArrayList<Object[]>(fields.length); for (Object[] schemas : fields) { list.add(schemas); } @@ -50,10 +49,12 @@ public class TestSchemaCompatibilityReaderFieldMissingDefaultValue { public Schema writer; @Parameter(2) public String details; + @Parameter(3) + public String location; @Test public void testReaderFieldMissingDefaultValueSchemas() throws Exception { validateIncompatibleSchemas(reader, writer, - SchemaIncompatibilityType.READER_FIELD_MISSING_DEFAULT_VALUE, details); + SchemaIncompatibilityType.READER_FIELD_MISSING_DEFAULT_VALUE, details, location); } } http://git-wip-us.apache.org/repos/asf/avro/blob/2df0775d/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityTypeMismatch.java ---------------------------------------------------------------------- diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityTypeMismatch.java b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityTypeMismatch.java index a8772a7..92db53c 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityTypeMismatch.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaCompatibilityTypeMismatch.java @@ -48,66 +48,66 @@ import org.junit.runners.Parameterized.Parameters; @RunWith(Parameterized.class) public class TestSchemaCompatibilityTypeMismatch { - @Parameters(name = "r: {0} | w: {1}") public static Iterable<Object[]> data() { Object[][] fields = { // - { NULL_SCHEMA, INT_SCHEMA, "reader type: NULL not compatible with writer type: INT" }, - { NULL_SCHEMA, LONG_SCHEMA, "reader type: NULL not compatible with writer type: LONG" }, + { NULL_SCHEMA, INT_SCHEMA, "reader type: NULL not compatible with writer type: INT", "/" }, + { NULL_SCHEMA, LONG_SCHEMA, "reader type: NULL not compatible with writer type: LONG", "/" }, - { BOOLEAN_SCHEMA, INT_SCHEMA, "reader type: BOOLEAN not compatible with writer type: INT" }, + { BOOLEAN_SCHEMA, INT_SCHEMA, "reader type: BOOLEAN not compatible with writer type: INT", "/" }, - { INT_SCHEMA, NULL_SCHEMA, "reader type: INT not compatible with writer type: NULL" }, - { INT_SCHEMA, BOOLEAN_SCHEMA, "reader type: INT not compatible with writer type: BOOLEAN" }, - { INT_SCHEMA, LONG_SCHEMA, "reader type: INT not compatible with writer type: LONG" }, - { INT_SCHEMA, FLOAT_SCHEMA, "reader type: INT not compatible with writer type: FLOAT" }, - { INT_SCHEMA, DOUBLE_SCHEMA, "reader type: INT not compatible with writer type: DOUBLE" }, + { INT_SCHEMA, NULL_SCHEMA, "reader type: INT not compatible with writer type: NULL", "/" }, + { INT_SCHEMA, BOOLEAN_SCHEMA, "reader type: INT not compatible with writer type: BOOLEAN", "/" }, + { INT_SCHEMA, LONG_SCHEMA, "reader type: INT not compatible with writer type: LONG", "/" }, + { INT_SCHEMA, FLOAT_SCHEMA, "reader type: INT not compatible with writer type: FLOAT", "/" }, + { INT_SCHEMA, DOUBLE_SCHEMA, "reader type: INT not compatible with writer type: DOUBLE", "/" }, - { LONG_SCHEMA, FLOAT_SCHEMA, "reader type: LONG not compatible with writer type: FLOAT" }, - { LONG_SCHEMA, DOUBLE_SCHEMA, "reader type: LONG not compatible with writer type: DOUBLE" }, + { LONG_SCHEMA, FLOAT_SCHEMA, "reader type: LONG not compatible with writer type: FLOAT", "/" }, + { LONG_SCHEMA, DOUBLE_SCHEMA, "reader type: LONG not compatible with writer type: DOUBLE", "/" }, { FLOAT_SCHEMA, DOUBLE_SCHEMA, - "reader type: FLOAT not compatible with writer type: DOUBLE" }, + "reader type: FLOAT not compatible with writer type: DOUBLE", "/" }, { DOUBLE_SCHEMA, STRING_SCHEMA, - "reader type: DOUBLE not compatible with writer type: STRING" }, + "reader type: DOUBLE not compatible with writer type: STRING", "/" }, { FIXED_4_BYTES, STRING_SCHEMA, - "reader type: FIXED not compatible with writer type: STRING" }, + "reader type: FIXED not compatible with writer type: STRING", "/" }, { STRING_SCHEMA, BOOLEAN_SCHEMA, - "reader type: STRING not compatible with writer type: BOOLEAN" }, - { STRING_SCHEMA, INT_SCHEMA, "reader type: STRING not compatible with writer type: INT" }, + "reader type: STRING not compatible with writer type: BOOLEAN", "/" }, + { STRING_SCHEMA, INT_SCHEMA, "reader type: STRING not compatible with writer type: INT", "/" }, - { BYTES_SCHEMA, NULL_SCHEMA, "reader type: BYTES not compatible with writer type: NULL" }, - { BYTES_SCHEMA, INT_SCHEMA, "reader type: BYTES not compatible with writer type: INT" }, + { BYTES_SCHEMA, NULL_SCHEMA, "reader type: BYTES not compatible with writer type: NULL", "/" }, + { BYTES_SCHEMA, INT_SCHEMA, "reader type: BYTES not compatible with writer type: INT", "/" }, - { A_INT_RECORD1, INT_SCHEMA, "reader type: RECORD not compatible with writer type: INT" }, + { A_INT_RECORD1, INT_SCHEMA, "reader type: RECORD not compatible with writer type: INT", "/" }, { INT_ARRAY_SCHEMA, LONG_ARRAY_SCHEMA, - "reader type: INT not compatible with writer type: LONG" }, + "reader type: INT not compatible with writer type: LONG", "/items" }, { INT_MAP_SCHEMA, INT_ARRAY_SCHEMA, - "reader type: MAP not compatible with writer type: ARRAY" }, + "reader type: MAP not compatible with writer type: ARRAY", "/" }, { INT_ARRAY_SCHEMA, INT_MAP_SCHEMA, - "reader type: ARRAY not compatible with writer type: MAP" }, + "reader type: ARRAY not compatible with writer type: MAP", "/" }, { INT_MAP_SCHEMA, LONG_MAP_SCHEMA, - "reader type: INT not compatible with writer type: LONG" }, + "reader type: INT not compatible with writer type: LONG", "/values" }, - { INT_SCHEMA, ENUM2_AB_SCHEMA, "reader type: INT not compatible with writer type: ENUM" }, - { ENUM2_AB_SCHEMA, INT_SCHEMA, "reader type: ENUM not compatible with writer type: INT" }, + { INT_SCHEMA, ENUM2_AB_SCHEMA, "reader type: INT not compatible with writer type: ENUM", "/" }, + { ENUM2_AB_SCHEMA, INT_SCHEMA, "reader type: ENUM not compatible with writer type: INT", "/" }, { FLOAT_SCHEMA, INT_LONG_FLOAT_DOUBLE_UNION_SCHEMA, - "reader type: FLOAT not compatible with writer type: DOUBLE" }, + "reader type: FLOAT not compatible with writer type: DOUBLE", "/" }, { LONG_SCHEMA, INT_FLOAT_UNION_SCHEMA, - "reader type: LONG not compatible with writer type: FLOAT" }, + "reader type: LONG not compatible with writer type: FLOAT", "/" }, { INT_SCHEMA, INT_FLOAT_UNION_SCHEMA, - "reader type: INT not compatible with writer type: FLOAT" }, + "reader type: INT not compatible with writer type: FLOAT", "/" }, { INT_LIST_RECORD, LONG_LIST_RECORD, - "reader type: INT not compatible with writer type: LONG" }, + "reader type: INT not compatible with writer type: LONG", "/fields/0/type" }, - { NULL_SCHEMA, INT_SCHEMA, "reader type: NULL not compatible with writer type: INT" } }; - List<Object[]> list = new ArrayList<>(fields.length); + { NULL_SCHEMA, INT_SCHEMA, "reader type: NULL not compatible with writer type: INT", "/" } + }; + List<Object[]> list = new ArrayList<Object[]>(fields.length); for (Object[] schemas : fields) { list.add(schemas); } @@ -120,9 +120,11 @@ public class TestSchemaCompatibilityTypeMismatch { public Schema writer; @Parameter(2) public String details; + @Parameter(3) + public String location; @Test public void testTypeMismatchSchemas() throws Exception { - validateIncompatibleSchemas(reader, writer, SchemaIncompatibilityType.TYPE_MISMATCH, details); + validateIncompatibleSchemas(reader, writer, SchemaIncompatibilityType.TYPE_MISMATCH, details, location); } } http://git-wip-us.apache.org/repos/asf/avro/blob/2df0775d/lang/java/avro/src/test/java/org/apache/avro/TestSchemas.java ---------------------------------------------------------------------- diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestSchemas.java b/lang/java/avro/src/test/java/org/apache/avro/TestSchemas.java index 264e7e6..d69dce9 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestSchemas.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestSchemas.java @@ -73,6 +73,12 @@ public class TestSchemas { Schema.createRecord("Record1", null, null, false); static final Schema A_INT_B_DINT_RECORD1 = Schema.createRecord("Record1", null, null, false); static final Schema A_DINT_B_DINT_RECORD1 = Schema.createRecord("Record1", null, null, false); + static final Schema A_DINT_B_DFIXED_4_BYTES_RECORD1 = Schema.createRecord("Record1", null, null, false); + static final Schema A_DINT_B_DFIXED_8_BYTES_RECORD1 = Schema.createRecord("Record1", null, null, false); + static final Schema A_DINT_B_DINT_STRING_UNION_RECORD1 = Schema.createRecord("Record1", null, null, false); + static final Schema A_DINT_B_DINT_UNION_RECORD1 = Schema.createRecord("Record1", null, null, false); + static final Schema A_DINT_B_DENUM_1_RECORD1 = Schema.createRecord("Record1", null, null, false); + static final Schema A_DINT_B_DENUM_2_RECORD1 = Schema.createRecord("Record1", null, null, false); static final Schema FIXED_4_BYTES = Schema.createFixed("Fixed", null, null, 4); static final Schema FIXED_8_BYTES = Schema.createFixed("Fixed", null, null, 8); @@ -89,6 +95,24 @@ public class TestSchemas { list(new Field("a", INT_SCHEMA, null, null), new Field("b", INT_SCHEMA, null, 0))); A_DINT_B_DINT_RECORD1 .setFields(list(new Field("a", INT_SCHEMA, null, 0), new Field("b", INT_SCHEMA, null, 0))); + A_DINT_B_DFIXED_4_BYTES_RECORD1.setFields(list( + new Field("a", INT_SCHEMA, null, 0), + new Field("b", FIXED_4_BYTES, null, 0))); + A_DINT_B_DFIXED_8_BYTES_RECORD1.setFields(list( + new Field("a", INT_SCHEMA, null, 0), + new Field("b", FIXED_8_BYTES, null, 0))); + A_DINT_B_DINT_STRING_UNION_RECORD1.setFields(list( + new Field("a", INT_SCHEMA, null, 0), + new Field("b", INT_STRING_UNION_SCHEMA, null, 0))); + A_DINT_B_DINT_UNION_RECORD1.setFields(list( + new Field("a", INT_SCHEMA, null, 0), + new Field("b", INT_UNION_SCHEMA, null, 0))); + A_DINT_B_DENUM_1_RECORD1.setFields(list( + new Field("a", INT_SCHEMA, null, 0), + new Field("b", ENUM1_AB_SCHEMA, null, 0))); + A_DINT_B_DENUM_2_RECORD1.setFields(list( + new Field("a", INT_SCHEMA, null, 0), + new Field("b", ENUM2_AB_SCHEMA, null, 0))); } // Recursive records @@ -124,7 +148,7 @@ public class TestSchemas { /** Borrowed from the Guava library. */ static <E> ArrayList<E> list(E... elements) { - final ArrayList<E> list = new ArrayList<>(); + final ArrayList<E> list = new ArrayList<E>(); Collections.addAll(list, elements); return list; }
