Repository: ignite Updated Branches: refs/heads/ignite-3098 2d5d1246d -> 74c051230
IGNITE-3098: UTF-16 surrogate pairs are not properly serialized by BinaryMarshaller Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/74c05123 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/74c05123 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/74c05123 Branch: refs/heads/ignite-3098 Commit: 74c051230b906897babbcb6553af60e2af1b44a0 Parents: 2d5d124 Author: Aleksei Scherbakov <[email protected]> Authored: Thu May 12 15:47:01 2016 +0300 Committer: Denis Magda <[email protected]> Committed: Thu May 12 15:47:01 2016 +0300 ---------------------------------------------------------------------- .../apache/ignite/IgniteSystemProperties.java | 5 + .../apache/ignite/internal/IgniteKernal.java | 6 + .../ignite/internal/IgniteNodeAttributes.java | 4 + .../ignite/internal/binary/BinaryUtils.java | 179 +++++++++++++++++-- .../internal/binary/BinaryWriterExImpl.java | 7 +- .../discovery/GridDiscoveryManager.java | 21 +++ .../ignite/spi/discovery/tcp/ServerImpl.java | 44 +++++ .../binary/BinaryMarshallerSelfTest.java | 47 ++++- .../GridDiscoveryManagerAttributesSelfTest.java | 63 +++++++ 9 files changed, 363 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/74c05123/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index b203a82..8e90e6c 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -391,6 +391,11 @@ public final class IgniteSystemProperties { "IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID"; /** + * Manages type of serialization mechanism for {@link String} that are marshalled/unmarshaller by BinaryMarshaller. + */ + public static final String IGNITE_USE_BINARY_STRING_SER_VER_2 = "IGNITE_USE_BINARY_STRING_SER_VER_2"; + + /** * If set to {@code true}, then default selected keys set is used inside * {@code GridNioServer} which lead to some extra garbage generation when * processing selected keys. http://git-wip-us.apache.org/repos/asf/ignite/blob/74c05123/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index 0f180b2..b2a8d25 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -84,6 +84,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.binary.BinaryEnumCache; import org.apache.ignite.internal.binary.BinaryMarshaller; +import org.apache.ignite.internal.binary.BinaryUtils; import org.apache.ignite.internal.cluster.ClusterGroupAdapter; import org.apache.ignite.internal.cluster.IgniteClusterEx; import org.apache.ignite.internal.managers.GridManager; @@ -179,6 +180,7 @@ import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALL import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK; import static org.apache.ignite.IgniteSystemProperties.IGNITE_STARVATION_CHECK_INTERVAL; import static org.apache.ignite.IgniteSystemProperties.IGNITE_SUCCESS_FILE; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_USE_BINARY_STRING_SER_VER_2; import static org.apache.ignite.IgniteSystemProperties.getBoolean; import static org.apache.ignite.IgniteSystemProperties.snapshot; import static org.apache.ignite.internal.GridKernalState.DISCONNECTED; @@ -207,6 +209,7 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTENT_ID; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PEER_CLASSLOADING; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PHY_RAM; @@ -1301,6 +1304,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable { add(ATTR_MARSHALLER_COMPACT_FOOTER, cfg.getBinaryConfiguration() == null ? BinaryConfiguration.DFLT_COMPACT_FOOTER : cfg.getBinaryConfiguration().isCompactFooter()); + + add(ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2, + getBoolean(IGNITE_USE_BINARY_STRING_SER_VER_2, BinaryUtils.USE_STR_SERIALIZATION_VER_2)); } add(ATTR_USER_NAME, System.getProperty("user.name")); http://git-wip-us.apache.org/repos/asf/ignite/blob/74c05123/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java index 3493eae..0e6a254 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java @@ -39,6 +39,10 @@ public final class IgniteNodeAttributes { /** Attribute for marshaller compact footers. */ public static final String ATTR_MARSHALLER_COMPACT_FOOTER = ATTR_PREFIX + ".marshaller.compactFooter"; + /** Internal attribute constant that controls which String serialization version to use. */ + public static final String ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2 = ATTR_PREFIX + + ".marshaller.utf8SerializationVer2"; + /** Internal attribute name constant. */ public static final String ATTR_JIT_NAME = ATTR_PREFIX + ".jit.name"; http://git-wip-us.apache.org/repos/asf/ignite/blob/74c05123/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java index c0202dd..0914814 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java @@ -46,6 +46,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListSet; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.binary.BinaryCollectionFactory; import org.apache.ignite.binary.BinaryInvalidTypeException; import org.apache.ignite.binary.BinaryMapFactory; @@ -63,6 +64,7 @@ import org.apache.ignite.lang.IgniteUuid; import org.jetbrains.annotations.Nullable; import org.jsr166.ConcurrentHashMap8; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_USE_BINARY_STRING_SER_VER_2; import static java.nio.charset.StandardCharsets.UTF_8; /** @@ -75,6 +77,10 @@ public class BinaryUtils { /** */ public static final Map<Byte, Class<?>> FLAG_TO_CLASS = new HashMap<>(); + /** */ + public static final boolean USE_STR_SERIALIZATION_VER_2 = IgniteSystemProperties.getBoolean( + IGNITE_USE_BINARY_STRING_SER_VER_2, true); + /** {@code true} if serialized value of this type cannot contain references to objects. */ private static final boolean[] PLAIN_TYPE_FLAG = new boolean[102]; @@ -415,7 +421,7 @@ public class BinaryUtils { break; case GridBinaryMarshaller.TIMESTAMP: - writer.doWriteTimestamp((Timestamp) val); + writer.doWriteTimestamp((Timestamp)val); break; @@ -614,7 +620,8 @@ public class BinaryUtils { } /** - * Attempts to create a new map of the same type as {@code map} has. Otherwise returns new {@code HashMap} instance. + * Attempts to create a new map of the same type as {@code map} has. Otherwise returns new {@code HashMap} + * instance. * * @param map Original map. * @return New map. @@ -648,8 +655,7 @@ public class BinaryUtils { } /** - * Attempts to create a new collection of the same known type. Will return null if collection type is - * unknown. + * Attempts to create a new collection of the same known type. Will return null if collection type is unknown. * * @param col Collection. * @return New empty collection. @@ -674,7 +680,8 @@ public class BinaryUtils { } /** - * Attempts to create a new set of the same type as {@code set} has. Otherwise returns new {@code HashSet} instance. + * Attempts to create a new set of the same type as {@code set} has. Otherwise returns new {@code HashSet} + * instance. * * @param set Original set. * @return New set. @@ -780,7 +787,7 @@ public class BinaryUtils { int len = length(in, start); - if (hasSchema(flags)){ + if (hasSchema(flags)) { // Schema exists. if (hasRaw(flags)) // Raw offset is set, it is at the very end of the object. @@ -1150,15 +1157,28 @@ public class BinaryUtils { * @return Value. */ public static String doReadString(BinaryInputStream in) { - if (!in.hasArray()) - return new String(doReadByteArray(in), UTF_8); + if (!in.hasArray()) { + byte[] arr = doReadByteArray(in); + + if (USE_STR_SERIALIZATION_VER_2) + return utf8BytesToStr(arr, 0, arr.length); + else + return new String(arr, UTF_8); + } int strLen = in.readInt(); int pos = in.position(); // String will copy necessary array part for us. - String res = new String(in.array(), pos, strLen, UTF_8); + String res; + + if (USE_STR_SERIALIZATION_VER_2) { + res = utf8BytesToStr(in.array(), pos, strLen); + } + else { + res = new String(in.array(), pos, strLen, UTF_8); + } in.position(pos + strLen); @@ -1485,7 +1505,7 @@ public class BinaryUtils { private static Object[] doReadBinaryEnumArray(BinaryInputStream in, BinaryContext ctx) { int len = in.readInt(); - Object[] arr = (Object[]) Array.newInstance(BinaryObject.class, len); + Object[] arr = (Object[])Array.newInstance(BinaryObject.class, len); for (int i = 0; i < len; i++) { byte flag = in.readByte(); @@ -1524,7 +1544,7 @@ public class BinaryUtils { throws BinaryObjectException { int len = in.readInt(); - Object[] arr = (Object[]) Array.newInstance(cls, len); + Object[] arr = (Object[])Array.newInstance(cls, len); for (int i = 0; i < len; i++) { byte flag = in.readByte(); @@ -2013,6 +2033,143 @@ public class BinaryUtils { } /** + * Reconstructs string from UTF-8 bytes. + * + * @param arr array Byte array. + * @param off offset Offset in the array. + * @param len length Byte array lenght. + * @return string Resulting string. + */ + public static String utf8BytesToStr(byte[] arr, int off, int len) { + int c, charArrCnt = 0, total = off + len; + int c2, c3; + char[] res = new char[len]; + + // try reading ascii + while (off < total) { + c = (int)arr[off] & 0xff; + + if (c > 127) + break; + + off++; + + res[charArrCnt++] = (char)c; + } + + // read other + while (off < total) { + c = (int)arr[off] & 0xff; + + switch (c >> 4) { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + /* 0xxxxxxx*/ + off++; + + res[charArrCnt++] = (char)c; + + break; + case 12: + case 13: + /* 110x xxxx 10xx xxxx*/ + off += 2; + + if (off > total) + throw new BinaryObjectException("Malformed input: partial character at end"); + + c2 = (int)arr[off - 1]; + + if ((c2 & 0xC0) != 0x80) + throw new BinaryObjectException("Malformed input around byte: " + off); + + res[charArrCnt++] = (char)(((c & 0x1F) << 6) | (c2 & 0x3F)); + + break; + case 14: + /* 1110 xxxx 10xx xxxx 10xx xxxx */ + off += 3; + + if (off > total) + throw new BinaryObjectException("Malformed input: partial character at end"); + + c2 = (int)arr[off - 2]; + + c3 = (int)arr[off - 1]; + + if (((c2 & 0xC0) != 0x80) || ((c3 & 0xC0) != 0x80)) + throw new BinaryObjectException("Malformed input around byte: " + (off - 1)); + + res[charArrCnt++] = (char)(((c & 0x0F) << 12) | + ((c2 & 0x3F) << 6) | + ((c3 & 0x3F) << 0)); + + break; + default: + /* 10xx xxxx, 1111 xxxx */ + throw new BinaryObjectException("Malformed input around byte: " + off); + } + } + + return len == charArrCnt ? new String(res) : new String(res, 0, charArrCnt); + } + + /** + * Converts the string into UTF-8 byte array considering special symbols like the surrogates. + * + * @param val String to convert. + * @return Resulting byte array. + */ + public static byte[] strToUtf8Bytes(String val) { + int strLen = val.length(); + int utfLen = 0; + int c, cnt; + + // Determine length of resulting byte array. + for (cnt = 0; cnt < strLen; cnt++) { + c = val.charAt(cnt); + + // ASCII + if (c <= 0x007F) + utfLen++; + // Special symbols (surrogates) + else if (c > 0x07FF) + utfLen += 3; + // The rest of the symbols. + else + utfLen += 2; + } + + byte[] arr = new byte[utfLen]; + + int position = 0; + + for (cnt = 0; cnt < strLen; cnt++) { + c = val.charAt(cnt); + + if (c <= 0x007F) + arr[position++] = (byte)c; + else if (c > 0x07FF) { + arr[position++] = (byte)(0xE0 | c >> 12 & 0x0F); + arr[position++] = (byte)(0x80 | c >> 6 & 0x3F); + arr[position++] = (byte)(0x80 | c & 0x3F); + } + else { + arr[position++] = (byte)(0xC0 | c >> 6 & 0x1F); + arr[position++] = (byte)(0x80 | c & 0x3F); + } + } + + return arr; + } + + /** * Enum type. */ private static class EnumType { http://git-wip-us.apache.org/repos/asf/ignite/blob/74c05123/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java index 8060a13..30710f4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java @@ -379,7 +379,12 @@ public class BinaryWriterExImpl implements BinaryWriter, BinaryRawWriterEx, Obje if (val == null) out.writeByte(GridBinaryMarshaller.NULL); else { - byte[] strArr = val.getBytes(UTF_8); + byte[] strArr; + + if (BinaryUtils.USE_STR_SERIALIZATION_VER_2) + strArr = BinaryUtils.strToUtf8Bytes(val); + else + strArr = val.getBytes(UTF_8); out.unsafeEnsure(1 + 4); out.unsafeWriteByte(GridBinaryMarshaller.STRING); http://git-wip-us.apache.org/repos/asf/ignite/blob/74c05123/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java index dc664a8..8f4c60f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java @@ -118,6 +118,7 @@ import org.jsr166.ConcurrentHashMap8; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_USE_BINARY_STRING_SER_VER_2; import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED; import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED; import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; @@ -128,6 +129,7 @@ import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DEPLOYMENT_MODE; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PEER_CLASSLOADING; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_USER_NAME; @@ -1003,6 +1005,11 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> { Boolean locMarshUseDfltSuid = locNode.attribute(ATTR_MARSHALLER_USE_DFLT_SUID); boolean locMarshUseDfltSuidBool = locMarshUseDfltSuid == null ? true : locMarshUseDfltSuid; + Boolean locMarshStrSerVer2 = locNode.attribute(ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2); + boolean locMarshStrSerVer2Bool = locMarshStrSerVer2 == null ? + false /* turned on and added to the attributes list by default only when BinaryMarshaller is used. */: + locMarshStrSerVer2; + boolean locDelayAssign = locNode.attribute(ATTR_LATE_AFFINITY_ASSIGNMENT); for (ClusterNode n : nodes) { @@ -1064,6 +1071,20 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> { ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']'); } + Boolean rmtMarshStrSerVer2 = n.attribute(ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2); + boolean rmtMarshStrSerVer2Bool = rmtMarshStrSerVer2 == null ? false : rmtMarshStrSerVer2; + + if (locMarshStrSerVer2Bool != rmtMarshStrSerVer2Bool) { + throw new IgniteCheckedException("Local node's " + IGNITE_USE_BINARY_STRING_SER_VER_2 + + " property value differs from remote node's value " + + "(to make sure all nodes in topology have identical marshaller settings, " + + "configure system property explicitly) " + + "[locMarshStrSerVer2=" + locMarshStrSerVer2 + ", rmtMarshStrSerVer2=" + rmtMarshStrSerVer2 + + ", locNodeAddrs=" + U.addressesAsString(locNode) + + ", rmtNodeAddrs=" + U.addressesAsString(n) + + ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']'); + } + boolean rmtLateAssign; if (n.version().compareToIgnoreTimestamp(CacheAffinitySharedManager.LATE_AFF_ASSIGN_SINCE) >= 0) http://git-wip-us.apache.org/repos/asf/ignite/blob/74c05123/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java index 84400ed..4c76632 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java @@ -133,6 +133,7 @@ import org.jsr166.ConcurrentHashMap8; import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISCOVERY_HISTORY_SIZE; import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_USE_BINARY_STRING_SER_VER_2; import static org.apache.ignite.IgniteSystemProperties.getInteger; import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; import static org.apache.ignite.events.EventType.EVT_NODE_JOINED; @@ -142,6 +143,7 @@ import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER; +import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID; import static org.apache.ignite.spi.IgnitePortProtocol.TCP; import static org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoverySpiState.AUTH_FAILED; @@ -3336,6 +3338,48 @@ class ServerImpl extends TcpDiscoveryImpl { return; } + // Validate String serialization mechanism used by the BinaryMarshaller. + Boolean locMarshStrSerialVer2 = locNode.attribute(ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2); + final boolean locMarshStrSerialVer2Bool = locMarshStrSerialVer2 != null ? locMarshStrSerialVer2 : false; + + Boolean rmtMarshStrSerialVer2 = node.attribute(ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2); + final boolean rmtMarshStrSerialVer2Bool = rmtMarshStrSerialVer2 != null ? rmtMarshStrSerialVer2 : false; + + if (locMarshStrSerialVer2Bool != rmtMarshStrSerialVer2Bool) { + utilityPool.submit( + new Runnable() { + @Override public void run() { + String errMsg = "Local node's " + IGNITE_USE_BINARY_STRING_SER_VER_2 + + " property value differs from remote node's value " + + "(to make sure all nodes in topology have identical marshaller settings, " + + "configure system property explicitly) " + + "[locMarshStrSerialVer2=" + locMarshStrSerialVer2 + + ", rmtMarshStrSerialVer2=" + rmtMarshStrSerialVer2 + + ", locNodeAddrs=" + U.addressesAsString(locNode) + + ", rmtNodeAddrs=" + U.addressesAsString(node) + + ", locNodeId=" + locNode.id() + ", rmtNodeId=" + msg.creatorNodeId() + ']'; + + String sndMsg = "Local node's " + IGNITE_USE_BINARY_STRING_SER_VER_2 + + " property value differs from remote node's value " + + "(to make sure all nodes in topology have identical marshaller settings, " + + "configure system property explicitly) " + + "[locMarshStrSerialVer2=" + rmtMarshStrSerialVer2 + + ", rmtMarshStrSerialVer2=" + locMarshStrSerialVer2 + + ", locNodeAddrs=" + U.addressesAsString(node) + ", locPort=" + node.discoveryPort() + + ", rmtNodeAddr=" + U.addressesAsString(locNode) + ", locNodeId=" + node.id() + + ", rmtNodeId=" + locNode.id() + ']'; + + nodeCheckError( + node, + errMsg, + sndMsg); + } + }); + + // Ignore join request. + return; + } + boolean rmtLateAssignBool; if (node.version().compareToIgnoreTimestamp(CacheAffinitySharedManager.LATE_AFF_ASSIGN_SINCE) >= 0) { http://git-wip-us.apache.org/repos/asf/ignite/blob/74c05123/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java index eefe66c..0b38c3b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryMarshallerSelfTest.java @@ -93,6 +93,8 @@ import static org.apache.ignite.internal.binary.streams.BinaryMemoryAllocator.IN import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertNotEquals; +import static java.nio.charset.StandardCharsets.UTF_8; + /** * Binary marshaller tests. */ @@ -180,7 +182,50 @@ public class BinaryMarshallerSelfTest extends GridCommonAbstractTest { * @throws Exception If failed. */ public void testString() throws Exception { - assertEquals("str", marshalUnmarshal("str")); + // Ascii check. + String str = "ascii0123456789"; + assertEquals(str, marshalUnmarshal(str)); + + byte[] bytes = str.getBytes(UTF_8); + assertEquals(str, BinaryUtils.utf8BytesToStr(bytes, 0, bytes.length)); + + bytes = BinaryUtils.strToUtf8Bytes(str); + assertEquals(str, new String(bytes, UTF_8)); + + // Extended symbols set check set. + str = "ççabcdкиÑиллиÑа"; + assertEquals(str, marshalUnmarshal(str)); + + bytes = str.getBytes(UTF_8); + assertEquals(str, BinaryUtils.utf8BytesToStr(bytes, 0, bytes.length)); + + bytes = BinaryUtils.strToUtf8Bytes(str); + assertEquals(str, new String(bytes, UTF_8)); + + // Special symbols check. + str = new String(new char[]{0xD800, 'ç', 0xD800, 0xD800, 0xDC00, 0xDFFF}); + assertEquals(str, marshalUnmarshal(str)); + + bytes = str.getBytes(UTF_8); + try { + BinaryUtils.utf8BytesToStr(bytes, 0, bytes.length); + assert false : "Mustn't be able to convert the string:" + str; + } + catch (BinaryObjectException e) { + //expected. + } + + bytes = BinaryUtils.strToUtf8Bytes(str); + assertNotEquals(str, new String(bytes, UTF_8)); + + str = new String(new char[]{55296}); + assertEquals(str, marshalUnmarshal(str)); + + bytes = str.getBytes(UTF_8); + assertNotEquals(str, BinaryUtils.utf8BytesToStr(bytes, 0, bytes.length)); + + bytes = BinaryUtils.strToUtf8Bytes(str); + assertNotEquals(str, new String(bytes, UTF_8)); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/74c05123/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java index 3a2f3ba..a1de38d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java @@ -21,6 +21,7 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.configuration.DeploymentMode; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.binary.BinaryMarshaller; import org.apache.ignite.marshaller.optimized.OptimizedMarshaller; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; @@ -28,6 +29,7 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import static org.apache.ignite.IgniteSystemProperties.IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_USE_BINARY_STRING_SER_VER_2; import static org.apache.ignite.configuration.DeploymentMode.CONTINUOUS; import static org.apache.ignite.configuration.DeploymentMode.SHARED; @@ -47,6 +49,9 @@ public abstract class GridDiscoveryManagerAttributesSelfTest extends GridCommonA /** */ private static boolean p2pEnabled; + /** */ + private static boolean binaryMarshallerEnabled; + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(gridName); @@ -54,6 +59,9 @@ public abstract class GridDiscoveryManagerAttributesSelfTest extends GridCommonA if (gridName.equals(getTestGridName(1))) cfg.setClientMode(true); + if (binaryMarshallerEnabled) + cfg.setMarshaller(new BinaryMarshaller()); + cfg.setIncludeProperties(PREFER_IPV4); cfg.setDeploymentMode(mode); cfg.setPeerClassLoadingEnabled(p2pEnabled); @@ -161,6 +169,61 @@ public abstract class GridDiscoveryManagerAttributesSelfTest extends GridCommonA } } + public void testUseStringSerVer2() throws Exception { + String old = System.getProperty(IGNITE_USE_BINARY_STRING_SER_VER_2); + + binaryMarshallerEnabled = true; + + try { + doTestUseStrSerVer2(Boolean.TRUE.toString(), Boolean.FALSE.toString(), true); + doTestUseStrSerVer2(Boolean.FALSE.toString(), Boolean.TRUE.toString(), true); + + doTestUseStrSerVer2(Boolean.TRUE.toString(), Boolean.TRUE.toString(), false); + doTestUseStrSerVer2(Boolean.FALSE.toString(), Boolean.FALSE.toString(), false); + } + finally { + if (old != null) + System.setProperty(IGNITE_USE_BINARY_STRING_SER_VER_2, old); + else + System.clearProperty(IGNITE_USE_BINARY_STRING_SER_VER_2); + + binaryMarshallerEnabled = false; + } + } + + /** + * @throws Exception If failed. + */ + private void doTestUseStrSerVer2(String first, String second, boolean fail) throws Exception { + try { + if (first != null) + System.setProperty(IGNITE_USE_BINARY_STRING_SER_VER_2, first); + else + System.clearProperty(IGNITE_USE_BINARY_STRING_SER_VER_2); + + startGrid(0); + + if (second != null) + System.setProperty(IGNITE_USE_BINARY_STRING_SER_VER_2, second); + else + System.clearProperty(IGNITE_USE_BINARY_STRING_SER_VER_2); + + try { + startGrid(1); + + if (fail) + fail("Node should not join"); + } + catch (Exception e) { + if (!fail) + fail("Node should join"); + } + } + finally { + stopAllGrids(); + } + } + /** * @throws Exception If failed. */
