IGNITE-3098: UTF-16 surrogate pairs are not properly serialized by BinaryMarshaller
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e49522b4 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e49522b4 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e49522b4 Branch: refs/heads/master Commit: e49522b4ec4fb67cd69a7101a1bc796746d82b4a Parents: a606f59 Author: Denis Magda <[email protected]> Authored: Tue May 17 15:46:27 2016 +0300 Committer: Denis Magda <[email protected]> Committed: Tue May 17 15:46:27 2016 +0300 ---------------------------------------------------------------------- .../apache/ignite/IgniteSystemProperties.java | 8 + .../apache/ignite/internal/IgniteKernal.java | 7 + .../ignite/internal/IgniteNodeAttributes.java | 4 + .../ignite/internal/binary/BinaryUtils.java | 176 +++++++++++++++++-- .../internal/binary/BinaryWriterExImpl.java | 7 +- .../discovery/GridDiscoveryManager.java | 21 +++ .../ignite/spi/discovery/tcp/ServerImpl.java | 44 +++++ .../binary/BinaryMarshallerSelfTest.java | 158 ++++++++++++----- .../GridDiscoveryManagerAttributesSelfTest.java | 63 +++++++ 9 files changed, 431 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/e49522b4/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index 35d190f..c28181e 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -394,6 +394,14 @@ public final class IgniteSystemProperties { "IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID"; /** + * Manages type of serialization mechanism for {@link String} that is marshalled/unmarshalled by BinaryMarshaller. + * Should be used for cases when a String contains a surrogate symbol without its pair one. This is frequently used + * in algorithms that encrypts data in String format. + */ + public static final String IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2 = + "IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2"; + + /** * If set to {@code true}, then default selected keys set is used inside * {@code GridNioServer} which lead to some extra garbage generation when * processing selected keys. http://git-wip-us.apache.org/repos/asf/ignite/blob/e49522b4/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index 0f180b2..18e5c62 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -84,6 +84,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.binary.BinaryEnumCache; import org.apache.ignite.internal.binary.BinaryMarshaller; +import org.apache.ignite.internal.binary.BinaryUtils; import org.apache.ignite.internal.cluster.ClusterGroupAdapter; import org.apache.ignite.internal.cluster.IgniteClusterEx; import org.apache.ignite.internal.managers.GridManager; @@ -179,6 +180,7 @@ import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALL import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK; import static org.apache.ignite.IgniteSystemProperties.IGNITE_STARVATION_CHECK_INTERVAL; import static org.apache.ignite.IgniteSystemProperties.IGNITE_SUCCESS_FILE; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2; import static org.apache.ignite.IgniteSystemProperties.getBoolean; import static org.apache.ignite.IgniteSystemProperties.snapshot; import static org.apache.ignite.internal.GridKernalState.DISCONNECTED; @@ -207,6 +209,7 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTENT_ID; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PEER_CLASSLOADING; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PHY_RAM; @@ -1301,6 +1304,10 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable { add(ATTR_MARSHALLER_COMPACT_FOOTER, cfg.getBinaryConfiguration() == null ? BinaryConfiguration.DFLT_COMPACT_FOOTER : cfg.getBinaryConfiguration().isCompactFooter()); + + add(ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2, + getBoolean(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2, + BinaryUtils.USE_STR_SERIALIZATION_VER_2)); } add(ATTR_USER_NAME, System.getProperty("user.name")); http://git-wip-us.apache.org/repos/asf/ignite/blob/e49522b4/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java index 3493eae..0e6a254 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java @@ -39,6 +39,10 @@ public final class IgniteNodeAttributes { /** Attribute for marshaller compact footers. */ public static final String ATTR_MARSHALLER_COMPACT_FOOTER = ATTR_PREFIX + ".marshaller.compactFooter"; + /** Internal attribute constant that controls which String serialization version to use. */ + public static final String ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2 = ATTR_PREFIX + + ".marshaller.utf8SerializationVer2"; + /** Internal attribute name constant. */ public static final String ATTR_JIT_NAME = ATTR_PREFIX + ".jit.name"; http://git-wip-us.apache.org/repos/asf/ignite/blob/e49522b4/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java index c0202dd..f1a7759 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java @@ -46,6 +46,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListSet; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.binary.BinaryCollectionFactory; import org.apache.ignite.binary.BinaryInvalidTypeException; import org.apache.ignite.binary.BinaryMapFactory; @@ -63,6 +64,7 @@ import org.apache.ignite.lang.IgniteUuid; import org.jetbrains.annotations.Nullable; import org.jsr166.ConcurrentHashMap8; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2; import static java.nio.charset.StandardCharsets.UTF_8; /** @@ -75,6 +77,10 @@ public class BinaryUtils { /** */ public static final Map<Byte, Class<?>> FLAG_TO_CLASS = new HashMap<>(); + /** */ + public static final boolean USE_STR_SERIALIZATION_VER_2 = IgniteSystemProperties.getBoolean( + IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2, false); + /** {@code true} if serialized value of this type cannot contain references to objects. */ private static final boolean[] PLAIN_TYPE_FLAG = new boolean[102]; @@ -415,7 +421,7 @@ public class BinaryUtils { break; case GridBinaryMarshaller.TIMESTAMP: - writer.doWriteTimestamp((Timestamp) val); + writer.doWriteTimestamp((Timestamp)val); break; @@ -614,7 +620,8 @@ public class BinaryUtils { } /** - * Attempts to create a new map of the same type as {@code map} has. Otherwise returns new {@code HashMap} instance. + * Attempts to create a new map of the same type as {@code map} has. Otherwise returns new {@code HashMap} + * instance. * * @param map Original map. * @return New map. @@ -648,8 +655,7 @@ public class BinaryUtils { } /** - * Attempts to create a new collection of the same known type. Will return null if collection type is - * unknown. + * Attempts to create a new collection of the same known type. Will return null if collection type is unknown. * * @param col Collection. * @return New empty collection. @@ -674,7 +680,8 @@ public class BinaryUtils { } /** - * Attempts to create a new set of the same type as {@code set} has. Otherwise returns new {@code HashSet} instance. + * Attempts to create a new set of the same type as {@code set} has. Otherwise returns new {@code HashSet} + * instance. * * @param set Original set. * @return New set. @@ -780,7 +787,7 @@ public class BinaryUtils { int len = length(in, start); - if (hasSchema(flags)){ + if (hasSchema(flags)) { // Schema exists. if (hasRaw(flags)) // Raw offset is set, it is at the very end of the object. @@ -1150,15 +1157,28 @@ public class BinaryUtils { * @return Value. */ public static String doReadString(BinaryInputStream in) { - if (!in.hasArray()) - return new String(doReadByteArray(in), UTF_8); + if (!in.hasArray()) { + byte[] arr = doReadByteArray(in); + + if (USE_STR_SERIALIZATION_VER_2) + return utf8BytesToStr(arr, 0, arr.length); + else + return new String(arr, UTF_8); + } int strLen = in.readInt(); int pos = in.position(); // String will copy necessary array part for us. - String res = new String(in.array(), pos, strLen, UTF_8); + String res; + + if (USE_STR_SERIALIZATION_VER_2) { + res = utf8BytesToStr(in.array(), pos, strLen); + } + else { + res = new String(in.array(), pos, strLen, UTF_8); + } in.position(pos + strLen); @@ -1485,7 +1505,7 @@ public class BinaryUtils { private static Object[] doReadBinaryEnumArray(BinaryInputStream in, BinaryContext ctx) { int len = in.readInt(); - Object[] arr = (Object[]) Array.newInstance(BinaryObject.class, len); + Object[] arr = (Object[])Array.newInstance(BinaryObject.class, len); for (int i = 0; i < len; i++) { byte flag = in.readByte(); @@ -1524,7 +1544,7 @@ public class BinaryUtils { throws BinaryObjectException { int len = in.readInt(); - Object[] arr = (Object[]) Array.newInstance(cls, len); + Object[] arr = (Object[])Array.newInstance(cls, len); for (int i = 0; i < len; i++) { byte flag = in.readByte(); @@ -2013,6 +2033,140 @@ public class BinaryUtils { } /** + * Reconstructs string from UTF-8 bytes. + * + * @param arr array Byte array. + * @param off offset Offset in the array. + * @param len length Byte array lenght. + * @return string Resulting string. + */ + public static String utf8BytesToStr(byte[] arr, int off, int len) { + int c, charArrCnt = 0, total = off + len; + int c2, c3; + char[] res = new char[len]; + + // try reading ascii + while (off < total) { + c = (int)arr[off] & 0xff; + + if (c > 127) + break; + + off++; + + res[charArrCnt++] = (char)c; + } + + // read other + while (off < total) { + c = (int)arr[off] & 0xff; + + switch (c >> 4) { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + /* 0xxxxxxx*/ + off++; + + res[charArrCnt++] = (char)c; + + break; + case 12: + case 13: + /* 110x xxxx 10xx xxxx*/ + off += 2; + + if (off > total) + throw new BinaryObjectException("Malformed input: partial character at end"); + + c2 = (int)arr[off - 1]; + + if ((c2 & 0xC0) != 0x80) + throw new BinaryObjectException("Malformed input around byte: " + off); + + res[charArrCnt++] = (char)(((c & 0x1F) << 6) | (c2 & 0x3F)); + + break; + case 14: + /* 1110 xxxx 10xx xxxx 10xx xxxx */ + off += 3; + + if (off > total) + throw new BinaryObjectException("Malformed input: partial character at end"); + + c2 = (int)arr[off - 2]; + + c3 = (int)arr[off - 1]; + + if (((c2 & 0xC0) != 0x80) || ((c3 & 0xC0) != 0x80)) + throw new BinaryObjectException("Malformed input around byte: " + (off - 1)); + + res[charArrCnt++] = (char)(((c & 0x0F) << 12) | + ((c2 & 0x3F) << 6) | + ((c3 & 0x3F) << 0)); + + break; + default: + /* 10xx xxxx, 1111 xxxx */ + throw new BinaryObjectException("Malformed input around byte: " + off); + } + } + + return len == charArrCnt ? new String(res) : new String(res, 0, charArrCnt); + } + + /** + * Converts the string into UTF-8 byte array considering special symbols like the surrogates. + * + * @param val String to convert. + * @return Resulting byte array. + */ + public static byte[] strToUtf8Bytes(String val) { + int strLen = val.length(); + int utfLen = 0; + int c, cnt; + + // Determine length of resulting byte array. + for (cnt = 0; cnt < strLen; cnt++) { + c = val.charAt(cnt); + + if (c >= 0x0001 && c <= 0x007F) + utfLen++; + else if (c > 0x07FF) + utfLen += 3; + else + utfLen += 2; + } + + byte[] arr = new byte[utfLen]; + + int position = 0; + + for (cnt = 0; cnt < strLen; cnt++) { + c = val.charAt(cnt); + + if (c >= 0x0001 && c <= 0x007F) + arr[position++] = (byte)c; + else if (c > 0x07FF) { + arr[position++] = (byte)(0xE0 | (c >> 12) & 0x0F); + arr[position++] = (byte)(0x80 | (c >> 6) & 0x3F); + arr[position++] = (byte)(0x80 | (c & 0x3F)); + } + else { + arr[position++] = (byte)(0xC0 | ((c >> 6) & 0x1F)); + arr[position++] = (byte)(0x80 | (c & 0x3F)); + } + } + + return arr; + } + + /** * Enum type. */ private static class EnumType { http://git-wip-us.apache.org/repos/asf/ignite/blob/e49522b4/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java index 8060a13..30710f4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java @@ -379,7 +379,12 @@ public class BinaryWriterExImpl implements BinaryWriter, BinaryRawWriterEx, Obje if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - byte[] strArr = val.getBytes(UTF_8); + byte[] strArr; + + if (BinaryUtils.USE_STR_SERIALIZATION_VER_2) + strArr = BinaryUtils.strToUtf8Bytes(val); + else + strArr = val.getBytes(UTF_8); out.unsafeEnsure(1 + 4); out.unsafeWriteByte(GridBinaryMarshaller.STRING); http://git-wip-us.apache.org/repos/asf/ignite/blob/e49522b4/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java index dc664a8..6aa4d54 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java @@ -118,6 +118,7 @@ import org.jsr166.ConcurrentHashMap8; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2; import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED; import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED; import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; @@ -128,6 +129,7 @@ import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DEPLOYMENT_MODE; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PEER_CLASSLOADING; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_USER_NAME; @@ -1003,6 +1005,11 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> { Boolean locMarshUseDfltSuid = locNode.attribute(ATTR_MARSHALLER_USE_DFLT_SUID); boolean locMarshUseDfltSuidBool = locMarshUseDfltSuid == null ? true : locMarshUseDfltSuid; + Boolean locMarshStrSerVer2 = locNode.attribute(ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2); + boolean locMarshStrSerVer2Bool = locMarshStrSerVer2 == null ? + false /* turned on and added to the attributes list by default only when BinaryMarshaller is used. */: + locMarshStrSerVer2; + boolean locDelayAssign = locNode.attribute(ATTR_LATE_AFFINITY_ASSIGNMENT); for (ClusterNode n : nodes) { @@ -1064,6 +1071,20 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> { ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']'); } + Boolean rmtMarshStrSerVer2 = n.attribute(ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2); + boolean rmtMarshStrSerVer2Bool = rmtMarshStrSerVer2 == null ? false : rmtMarshStrSerVer2; + + if (locMarshStrSerVer2Bool != rmtMarshStrSerVer2Bool) { + throw new IgniteCheckedException("Local node's " + IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2 + + " property value differs from remote node's value " + + "(to make sure all nodes in topology have identical marshaller settings, " + + "configure system property explicitly) " + + "[locMarshStrSerVer2=" + locMarshStrSerVer2 + ", rmtMarshStrSerVer2=" + rmtMarshStrSerVer2 + + ", locNodeAddrs=" + U.addressesAsString(locNode) + + ", rmtNodeAddrs=" + U.addressesAsString(n) + + ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']'); + } + boolean rmtLateAssign; if (n.version().compareToIgnoreTimestamp(CacheAffinitySharedManager.LATE_AFF_ASSIGN_SINCE) >= 0) http://git-wip-us.apache.org/repos/asf/ignite/blob/e49522b4/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java index 84400ed..a4c7276 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java @@ -133,6 +133,7 @@ import org.jsr166.ConcurrentHashMap8; import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISCOVERY_HISTORY_SIZE; import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2; import static org.apache.ignite.IgniteSystemProperties.getInteger; import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; import static org.apache.ignite.events.EventType.EVT_NODE_JOINED; @@ -142,6 +143,7 @@ import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID; import static org.apache.ignite.spi.IgnitePortProtocol.TCP; import static org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoverySpiState.AUTH_FAILED; @@ -3336,6 +3338,48 @@ class ServerImpl extends TcpDiscoveryImpl { return; } + // Validate String serialization mechanism used by the BinaryMarshaller. + final Boolean locMarshStrSerialVer2 = locNode.attribute(ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2); + final boolean locMarshStrSerialVer2Bool = locMarshStrSerialVer2 != null ? locMarshStrSerialVer2 : false; + + final Boolean rmtMarshStrSerialVer2 = node.attribute(ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2); + final boolean rmtMarshStrSerialVer2Bool = rmtMarshStrSerialVer2 != null ? rmtMarshStrSerialVer2 : false; + + if (locMarshStrSerialVer2Bool != rmtMarshStrSerialVer2Bool) { + utilityPool.submit( + new Runnable() { + @Override public void run() { + String errMsg = "Local node's " + IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2 + + " property value differs from remote node's value " + + "(to make sure all nodes in topology have identical marshaller settings, " + + "configure system property explicitly) " + + "[locMarshStrSerialVer2=" + locMarshStrSerialVer2 + + ", rmtMarshStrSerialVer2=" + rmtMarshStrSerialVer2 + + ", locNodeAddrs=" + U.addressesAsString(locNode) + + ", rmtNodeAddrs=" + U.addressesAsString(node) + + ", locNodeId=" + locNode.id() + ", rmtNodeId=" + msg.creatorNodeId() + ']'; + + String sndMsg = "Local node's " + IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2 + + " property value differs from remote node's value " + + "(to make sure all nodes in topology have identical marshaller settings, " + + "configure system property explicitly) " + + "[locMarshStrSerialVer2=" + rmtMarshStrSerialVer2 + + ", rmtMarshStrSerialVer2=" + locMarshStrSerialVer2 + + ", locNodeAddrs=" + U.addressesAsString(node) + ", locPort=" + node.discoveryPort() + + ", rmtNodeAddr=" + U.addressesAsString(locNode) + ", locNodeId=" + node.id() + + ", rmtNodeId=" + locNode.id() + ']'; + + nodeCheckError( + node, + errMsg, + sndMsg); + } + }); + + // Ignore join request. + return; + } + boolean rmtLateAssignBool; if (node.version().compareToIgnoreTimestamp(CacheAffinitySharedManager.LATE_AFF_ASSIGN_SINCE) >= 0) { http://git-wip-us.apache.org/repos/asf/ignite/blob/e49522b4/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java index eefe66c..936b1e6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java @@ -26,6 +26,7 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationHandler; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.lang.reflect.Modifier; import java.lang.reflect.Proxy; import java.math.BigDecimal; import java.math.BigInteger; @@ -89,10 +90,13 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jsr166.ConcurrentHashMap8; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2; import static org.apache.ignite.internal.binary.streams.BinaryMemoryAllocator.INSTANCE; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertNotEquals; +import static java.nio.charset.StandardCharsets.UTF_8; + /** * Binary marshaller tests. */ @@ -179,8 +183,69 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { /** * @throws Exception If failed. */ - public void testString() throws Exception { - assertEquals("str", marshalUnmarshal("str")); + public void testStringVer1() throws Exception { + doTestString(false); + } + + /** + * @throws Exception If failed. + */ + public void testStringVer2() throws Exception { + doTestString(true); + } + + /** + * @throws Exception If failed + */ + private void doTestString(boolean ver2) throws Exception { + // Ascii check. + String str = "ascii0123456789"; + assertEquals(str, marshalUnmarshal(str)); + + byte[] bytes = str.getBytes(UTF_8); + assertEquals(str, BinaryUtils.utf8BytesToStr(bytes, 0, bytes.length)); + + bytes = BinaryUtils.strToUtf8Bytes(str); + assertEquals(str, new String(bytes, UTF_8)); + + // Extended symbols set check set. + str = "ççabcdкиÑиллиÑа"; + assertEquals(str, marshalUnmarshal(str)); + + bytes = str.getBytes(UTF_8); + assertEquals(str, BinaryUtils.utf8BytesToStr(bytes, 0, bytes.length)); + + bytes = BinaryUtils.strToUtf8Bytes(str); + assertEquals(str, new String(bytes, UTF_8)); + + // Special symbols check. + str = new String(new char[] {0xD800, 'ç', 0xD800, 0xD800, 0xDC00, 0xDFFF}); + if (ver2) { + bytes = BinaryUtils.strToUtf8Bytes(str); + assertEquals(str, BinaryUtils.utf8BytesToStr(bytes, 0, bytes.length)); + } + else + assertNotEquals(str, marshalUnmarshal(str)); + + str = new String(new char[] {55296}); + if (ver2) { + bytes = BinaryUtils.strToUtf8Bytes(str); + assertEquals(str, BinaryUtils.utf8BytesToStr(bytes, 0, bytes.length)); + } + else + assertNotEquals(str, marshalUnmarshal(str)); + + bytes = str.getBytes(UTF_8); + assertNotEquals(str, new String(bytes, UTF_8)); + + bytes = str.getBytes(UTF_8); + assertNotEquals(str, BinaryUtils.utf8BytesToStr(bytes, 0, bytes.length)); + + str = new String(new char[] {0xD801, 0xDC37}); + assertEquals(str, marshalUnmarshal(str)); + + bytes = str.getBytes(UTF_8); + assertEquals(str, new String(bytes, UTF_8)); } /** @@ -291,7 +356,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { * @throws Exception If failed. */ public void testDecimalArray() throws Exception { - BigDecimal[] arr = new BigDecimal[] {BigDecimal.ZERO, BigDecimal.ONE, BigDecimal.TEN} ; + BigDecimal[] arr = new BigDecimal[] {BigDecimal.ZERO, BigDecimal.ONE, BigDecimal.TEN}; assertArrayEquals(arr, marshalUnmarshal(arr)); } @@ -912,9 +977,9 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { */ public void testClassWithoutPublicConstructor() throws Exception { BinaryMarshaller marsh = binaryMarshaller(Arrays.asList( - new BinaryTypeConfiguration(NoPublicConstructor.class.getName()), - new BinaryTypeConfiguration(NoPublicDefaultConstructor.class.getName()), - new BinaryTypeConfiguration(ProtectedConstructor.class.getName())) + new BinaryTypeConfiguration(NoPublicConstructor.class.getName()), + new BinaryTypeConfiguration(NoPublicDefaultConstructor.class.getName()), + new BinaryTypeConfiguration(ProtectedConstructor.class.getName())) ); NoPublicConstructor npc = new NoPublicConstructor(); @@ -1034,8 +1099,10 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { @Override public int fieldId(int typeId, String fieldName) { assert typeId == 44444; - if ("val1".equals(fieldName)) return 55555; - else if ("val2".equals(fieldName)) return 66666; + if ("val1".equals(fieldName)) + return 55555; + else if ("val2".equals(fieldName)) + return 66666; assert false : "Unknown field: " + fieldName; @@ -1403,14 +1470,14 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryMarshaller marsh = binaryMarshaller(new BinaryBasicNameMapper(true), new BinaryBasicIdMapper(true), Arrays.asList( - new BinaryTypeConfiguration(Key.class.getName()), - new BinaryTypeConfiguration("org.gridgain.NonExistentClass3"), - new BinaryTypeConfiguration("NonExistentClass4"), - customType1, - customType2, - customType3, - customType4 - )); + new BinaryTypeConfiguration(Key.class.getName()), + new BinaryTypeConfiguration("org.gridgain.NonExistentClass3"), + new BinaryTypeConfiguration("NonExistentClass4"), + customType1, + customType2, + customType3, + customType4 + )); BinaryContext ctx = binaryContext(marsh); @@ -1591,6 +1658,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { // BinaryIdMapper.typeId() contract. assertEquals("nonexistentclass0".hashCode(), ctx.typeId("NonExistentClass0")); } + /** * @throws Exception If failed. */ @@ -1693,7 +1761,6 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { assertEquals(992, ctx.typeId("org.gridgain.NonExistentClass3")); assertEquals(993, ctx.typeId("NonExistentClass4")); - // Custom types. assertEquals(300, ctx.typeId(Value.class.getName())); assertEquals(400, ctx.typeId("org.gridgain.NonExistentClass1")); @@ -1990,7 +2057,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryObject po = marshal(obj, marsh); - BinaryObject copy = copy(po, F.<String, Object>asMap("bArr", new byte[]{1, 2, 3})); + BinaryObject copy = copy(po, F.<String, Object>asMap("bArr", new byte[] {1, 2, 3})); assertArrayEquals(new byte[] {1, 2, 3}, copy.<byte[]>field("bArr")); @@ -2027,7 +2094,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryObject po = marshal(obj, marsh); - BinaryObject copy = copy(po, F.<String, Object>asMap("sArr", new short[]{1, 2, 3})); + BinaryObject copy = copy(po, F.<String, Object>asMap("sArr", new short[] {1, 2, 3})); assertArrayEquals(new short[] {1, 2, 3}, copy.<short[]>field("sArr")); @@ -2048,7 +2115,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryObject po = marshal(obj, marsh); - BinaryObject copy = copy(po, F.<String, Object>asMap("iArr", new int[]{1, 2, 3})); + BinaryObject copy = copy(po, F.<String, Object>asMap("iArr", new int[] {1, 2, 3})); assertArrayEquals(new int[] {1, 2, 3}, copy.<int[]>field("iArr")); @@ -2069,7 +2136,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryObject po = marshal(obj, marsh); - BinaryObject copy = copy(po, F.<String, Object>asMap("lArr", new long[]{1, 2, 3})); + BinaryObject copy = copy(po, F.<String, Object>asMap("lArr", new long[] {1, 2, 3})); assertArrayEquals(new long[] {1, 2, 3}, copy.<long[]>field("lArr")); @@ -2090,7 +2157,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryObject po = marshal(obj, marsh); - BinaryObject copy = copy(po, F.<String, Object>asMap("fArr", new float[]{1, 2, 3})); + BinaryObject copy = copy(po, F.<String, Object>asMap("fArr", new float[] {1, 2, 3})); assertArrayEquals(new float[] {1, 2, 3}, copy.<float[]>field("fArr"), 0); @@ -2111,7 +2178,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryObject po = marshal(obj, marsh); - BinaryObject copy = copy(po, F.<String, Object>asMap("dArr", new double[]{1, 2, 3})); + BinaryObject copy = copy(po, F.<String, Object>asMap("dArr", new double[] {1, 2, 3})); assertArrayEquals(new double[] {1, 2, 3}, copy.<double[]>field("dArr"), 0); @@ -2132,13 +2199,13 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryObject po = marshal(obj, marsh); - BinaryObject copy = copy(po, F.<String, Object>asMap("cArr", new char[]{1, 2, 3})); + BinaryObject copy = copy(po, F.<String, Object>asMap("cArr", new char[] {1, 2, 3})); - assertArrayEquals(new char[]{1, 2, 3}, copy.<char[]>field("cArr")); + assertArrayEquals(new char[] {1, 2, 3}, copy.<char[]>field("cArr")); SimpleObject obj0 = copy.deserialize(); - assertArrayEquals(new char[]{1, 2, 3}, obj0.cArr); + assertArrayEquals(new char[] {1, 2, 3}, obj0.cArr); } /** @@ -2153,7 +2220,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { BinaryObject po = marshal(obj, marsh); - BinaryObject copy = copy(po, F.<String, Object>asMap("strArr", new String[]{"str1", "str2"})); + BinaryObject copy = copy(po, F.<String, Object>asMap("strArr", new String[] {"str1", "str2"})); assertArrayEquals(new String[] {"str1", "str2"}, copy.<String[]>field("strArr")); @@ -2211,13 +2278,13 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { map.put("str", "str555"); map.put("inner", newObj); - map.put("bArr", new byte[]{6, 7, 9}); + map.put("bArr", new byte[] {6, 7, 9}); BinaryObject copy = copy(po, map); assertEquals("str555", copy.<String>field("str")); assertEquals(newObj, copy.<BinaryObject>field("inner").deserialize()); - assertArrayEquals(new byte[]{6, 7, 9}, copy.<byte[]>field("bArr")); + assertArrayEquals(new byte[] {6, 7, 9}, copy.<byte[]>field("bArr")); SimpleObject obj0 = copy.deserialize(); @@ -2440,7 +2507,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { DecimalReflective obj1 = new DecimalReflective(); obj1.val = BigDecimal.ZERO; - obj1.valArr = new BigDecimal[] { BigDecimal.ONE, BigDecimal.TEN }; + obj1.valArr = new BigDecimal[] {BigDecimal.ONE, BigDecimal.TEN}; BinaryObjectImpl portObj = marshal(obj1, marsh); @@ -2454,9 +2521,9 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { DecimalMarshalAware obj2 = new DecimalMarshalAware(); obj2.val = BigDecimal.ZERO; - obj2.valArr = new BigDecimal[] { BigDecimal.ONE, BigDecimal.TEN.negate() }; + obj2.valArr = new BigDecimal[] {BigDecimal.ONE, BigDecimal.TEN.negate()}; obj2.rawVal = BigDecimal.TEN; - obj2.rawValArr = new BigDecimal[] { BigDecimal.ZERO, BigDecimal.ONE }; + obj2.rawValArr = new BigDecimal[] {BigDecimal.ZERO, BigDecimal.ONE}; portObj = marshal(obj2, marsh); @@ -2788,7 +2855,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { /** * Some non-serializable class. */ - @SuppressWarnings( {"PublicField","TransientFieldInNonSerializableClass","FieldMayBeStatic"}) + @SuppressWarnings({"PublicField", "TransientFieldInNonSerializableClass", "FieldMayBeStatic"}) private static class NonSerializableA { /** */ private final long longVal = 0x33445566778899AAL; @@ -2797,7 +2864,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { protected Short shortVal = (short)0xAABB; /** */ - public String[] strArr = {"AA","BB"}; + public String[] strArr = {"AA", "BB"}; /** */ public boolean flag1 = true; @@ -2821,7 +2888,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { * @param strArr Array. * @param shortVal Short value. */ - @SuppressWarnings( {"UnusedDeclaration"}) + @SuppressWarnings({"UnusedDeclaration"}) private NonSerializableA(@Nullable String[] strArr, @Nullable Short shortVal) { // No-op. } @@ -2834,7 +2901,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { assertEquals(shortVal.shortValue(), (short)0xAABB); - assertTrue(Arrays.equals(strArr, new String[] {"AA","BB"})); + assertTrue(Arrays.equals(strArr, new String[] {"AA", "BB"})); assertEquals(0, intVal); @@ -2849,7 +2916,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { /** * Some non-serializable class. */ - @SuppressWarnings( {"PublicField","TransientFieldInNonSerializableClass","PackageVisibleInnerClass"}) + @SuppressWarnings({"PublicField", "TransientFieldInNonSerializableClass", "PackageVisibleInnerClass"}) static class NonSerializableB extends NonSerializableA { /** */ public Short shortValue = 0x1122; @@ -2893,7 +2960,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { /** * Some non-serializable class. */ - @SuppressWarnings( {"TransientFieldInNonSerializableClass","PublicField"}) + @SuppressWarnings({"TransientFieldInNonSerializableClass", "PublicField"}) private static class NonSerializable extends NonSerializableB { /** */ private int idVal = -17; @@ -2918,7 +2985,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { * * @param aVal Unused. */ - @SuppressWarnings( {"UnusedDeclaration"}) + @SuppressWarnings({"UnusedDeclaration"}) private NonSerializable(NonSerializableA aVal) { } @@ -3108,7 +3175,8 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { /** * */ - protected BinaryMarshaller binaryMarshaller(BinaryNameMapper nameMapper, BinaryIdMapper mapper, Collection<BinaryTypeConfiguration> cfgs) + protected BinaryMarshaller binaryMarshaller(BinaryNameMapper nameMapper, BinaryIdMapper mapper, + Collection<BinaryTypeConfiguration> cfgs) throws IgniteCheckedException { return binaryMarshaller(nameMapper, mapper, null, cfgs); } @@ -3250,7 +3318,6 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { outer.inner = inner; outer.bdArr = new BigDecimal[] {new BigDecimal(5000), BigDecimal.TEN}; - outer.col.add("str4"); outer.col.add("str5"); outer.col.add("str6"); @@ -4329,7 +4396,9 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { // No-op. } - /**n + /** + * n + * * @param key Key. */ private Key(int key) { @@ -4440,7 +4509,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { */ private static class ProtectedConstructor { /** - * Protected constructor. + * Protected constructor. */ protected ProtectedConstructor() { // No-op. @@ -4496,8 +4565,7 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { /** */ private String s; - /** Initializer. */ - { + /** Initializer. */ { StringBuilder builder = new StringBuilder(); for (int i = 0; i < 10000; i++) { http://git-wip-us.apache.org/repos/asf/ignite/blob/e49522b4/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java index 3a2f3ba..28380df 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java @@ -21,6 +21,7 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.configuration.DeploymentMode; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.binary.BinaryMarshaller; import org.apache.ignite.marshaller.optimized.OptimizedMarshaller; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; @@ -28,6 +29,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2; import static org.apache.ignite.configuration.DeploymentMode.CONTINUOUS; import static org.apache.ignite.configuration.DeploymentMode.SHARED; @@ -47,6 +49,9 @@ public abstract class GridDiscoveryManagerAttributesSelfTest extends GridCommonA /** */ private static boolean p2pEnabled; + /** */ + private static boolean binaryMarshallerEnabled; + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(gridName); @@ -54,6 +59,9 @@ public abstract class GridDiscoveryManagerAttributesSelfTest extends GridCommonA if (gridName.equals(getTestGridName(1))) cfg.setClientMode(true); + if (binaryMarshallerEnabled) + cfg.setMarshaller(new BinaryMarshaller()); + cfg.setIncludeProperties(PREFER_IPV4); cfg.setDeploymentMode(mode); cfg.setPeerClassLoadingEnabled(p2pEnabled); @@ -161,6 +169,61 @@ public abstract class GridDiscoveryManagerAttributesSelfTest extends GridCommonA } } + public void testUseStringSerVer2() throws Exception { + String old = System.getProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2); + + binaryMarshallerEnabled = true; + + try { + doTestUseStrSerVer2(Boolean.TRUE.toString(), Boolean.FALSE.toString(), true); + doTestUseStrSerVer2(Boolean.FALSE.toString(), Boolean.TRUE.toString(), true); + + doTestUseStrSerVer2(Boolean.TRUE.toString(), Boolean.TRUE.toString(), false); + doTestUseStrSerVer2(Boolean.FALSE.toString(), Boolean.FALSE.toString(), false); + } + finally { + if (old != null) + System.setProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2, old); + else + System.clearProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2); + + binaryMarshallerEnabled = false; + } + } + + /** + * @throws Exception If failed. + */ + private void doTestUseStrSerVer2(String first, String second, boolean fail) throws Exception { + try { + if (first != null) + System.setProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2, first); + else + System.clearProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2); + + startGrid(0); + + if (second != null) + System.setProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2, second); + else + System.clearProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2); + + try { + startGrid(1); + + if (fail) + fail("Node should not join"); + } + catch (Exception e) { + if (!fail) + fail("Node should join"); + } + } + finally { + stopAllGrids(); + } + } + /** * @throws Exception If failed. */
