This is an automated email from the ASF dual-hosted git repository. matthiasblaesing pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/netbeans.git
The following commit(s) were added to refs/heads/master by this push: new 3b8dea5 [NETBEANS-5181] NbBundle only accepts ISO-8859-1 while UTF-8 is default since JDK9 new 396d4be Merge pull request #2633 from jjazzboss/NETBEANS-5181-NbBundle 3b8dea5 is described below commit 3b8dea5f27a889a42f6356efd23a602418d6c5d3 Author: Jerome Lelasseux <j...@jjazzlab.com> AuthorDate: Fri Jan 1 21:20:47 2021 +0100 [NETBEANS-5181] NbBundle only accepts ISO-8859-1 while UTF-8 is default since JDK9 Change NbBundle to read Bundle files using UTF-1 encoding by default, and automatically switch to ISO-8859-1 if UTF-1 decoding fails. System property java.util.PropertyResourceBundle.encoding can can alter the behavior like for PropertyResourceBundle (from JDK9). --- .../src/org/openide/util/NbBundle.java | 106 +++++++++++++++++- .../unit/src/org/openide/util/NbBundleTest.java | 120 ++++++++++++++++----- 2 files changed, 199 insertions(+), 27 deletions(-) diff --git a/platform/openide.util/src/org/openide/util/NbBundle.java b/platform/openide.util/src/org/openide/util/NbBundle.java index 77b3473..e072141 100644 --- a/platform/openide.util/src/org/openide/util/NbBundle.java +++ b/platform/openide.util/src/org/openide/util/NbBundle.java @@ -21,6 +21,7 @@ package org.openide.util; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -28,6 +29,14 @@ import java.lang.annotation.Target; import java.lang.ref.Reference; import java.lang.ref.WeakReference; import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.Enumeration; @@ -68,7 +77,10 @@ public class NbBundle extends Object { private static final boolean USE_DEBUG_LOADER = Boolean.getBoolean("org.openide.util.NbBundle.DEBUG"); // NOI18N private static String brandingToken = null; - + + private static final UtfThenIsoCharset utfThenIsoCharset = new UtfThenIsoCharset(false); + private static final UtfThenIsoCharset utfThenIsoCharsetOnlyUTF8 = new UtfThenIsoCharset(true); + /** * Cache of URLs for localized files. * Keeps only weak references to the class loaders. @@ -538,8 +550,16 @@ public class NbBundle extends Object { (loader != null ? loader.getResourceAsStream(res) : ClassLoader.getSystemResourceAsStream(res)) : u.openStream(); + // #NETBEANS-5181 + String encoding = System.getProperty("java.util.PropertyResourceBundle.encoding"); + UtfThenIsoCharset charset = "UTF-8".equals(encoding) ? utfThenIsoCharsetOnlyUTF8 : utfThenIsoCharset; + InputStreamReader reader = new InputStreamReader(is, + "ISO-8859-1".equals(encoding) + ? StandardCharsets.ISO_8859_1.newDecoder() + : charset.newDecoder()); + try { - p.load(is); + p.load(reader); } finally { is.close(); } @@ -1458,4 +1478,86 @@ public class NbBundle extends Object { } } + + + /** + * Local charset to decode using UTF-8 by default, but automatically switching to ISO-8859-1 if UTF-8 decoding fails. + * + */ + static private class UtfThenIsoCharset extends Charset { + + private final boolean onlyUTF8; + + /** + * + * @param acceptOnlyUTF8 If true there is no automatic switch to ISO-8859-1 if UTF-8 decoding fails. + */ + public UtfThenIsoCharset(boolean acceptOnlyUTF8) { + super(UtfThenIsoCharset.class.getCanonicalName(), null); + this.onlyUTF8 = acceptOnlyUTF8; + } + + @Override + public boolean contains(Charset arg0) { + return this.equals(arg0); + } + + @Override + public CharsetDecoder newDecoder() { + return new UtfThenIsoDecoder(this, 1.0f, 1.0f); + } + + @Override + public CharsetEncoder newEncoder() { + throw new UnsupportedOperationException("Not supported yet."); + } + + private final class UtfThenIsoDecoder extends CharsetDecoder { + + private CharsetDecoder decoderUTF; + private CharsetDecoder decoderISO; // Not null means we switched to ISO + + protected UtfThenIsoDecoder(Charset cs, float averageCharsPerByte, float maxCharsPerByte) { + super(cs, averageCharsPerByte, maxCharsPerByte); + + decoderUTF = StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) // We want to be informed of this error + .onUnmappableCharacter(CodingErrorAction.REPORT); // We want to be informed of this error + } + + @Override + protected CoderResult decodeLoop(ByteBuffer in, CharBuffer out) { + + if (decoderISO != null) { + // No turning back once we've switched to ISO + return decoderISO.decode(in, out, false); + } + + // To rewind if need to retry with ISO decoding + in.mark(); + out.mark(); + + + // UTF decoding + CoderResult cr = decoderUTF.decode(in, out, false); + if (cr.isUnderflow() || cr.isOverflow()) { + // Normal results + return cr; + } + + // If we're here there was a malformed-input or unmappable-character error with the UTF decoding + if (UtfThenIsoCharset.this.onlyUTF8) { + // But can't switch to ISO + return cr; + } + + // Switch to ISO + in.reset(); + out.reset(); + decoderISO = StandardCharsets.ISO_8859_1.newDecoder(); + return decoderISO.decode(in, out, false); + } + } + } + } diff --git a/platform/openide.util/test/unit/src/org/openide/util/NbBundleTest.java b/platform/openide.util/test/unit/src/org/openide/util/NbBundleTest.java index fce554a..444e042 100644 --- a/platform/openide.util/test/unit/src/org/openide/util/NbBundleTest.java +++ b/platform/openide.util/test/unit/src/org/openide/util/NbBundleTest.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.openide.util; import java.io.ByteArrayInputStream; @@ -40,8 +39,10 @@ import java.util.ResourceBundle; import java.util.TreeMap; import java.util.jar.Attributes; import junit.framework.TestCase; -import org.openide.util.NbBundle; -import org.openide.util.NbCollections; +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertNotNull; +import static junit.framework.TestCase.fail; +import static org.junit.Assert.assertNotEquals; import org.openide.util.base.BundleClass; // XXX testGetClassBundle @@ -63,6 +64,7 @@ public class NbBundleTest extends TestCase { NbBundle.setBranding(null); NbBundle.localizedFileCache.clear(); NbBundle.bundleCache.clear(); + System.clearProperty("java.util.PropertyResourceBundle.encoding"); } public static void testLocalizingSuffixes() throws Exception { @@ -75,6 +77,7 @@ public class NbBundleTest extends TestCase { Locale.setDefault(Locale.JAPAN); assertEquals("_f4j_ce_ja_JP,_f4j_ce_ja,_f4j_ce,_f4j_ja_JP,_f4j_ja,_f4j,_ja_JP,_ja,", locSuff()); } + private static String locSuff() { StringBuffer b = new StringBuffer(); boolean first = true; @@ -91,7 +94,7 @@ public class NbBundleTest extends TestCase { } public void testGetBundle() throws Exception { - ResourceBundle rb = NbBundle.getBundle("foo.Bundle", Locale.ENGLISH, fixedLoader("foo/Bundle.properties:k=v")); + ResourceBundle rb = NbBundle.getBundle("foo.Bundle", Locale.ENGLISH, fixedLoader(false, "foo/Bundle.properties:k=v")); assertEquals("v", rb.getString("k")); try { rb.getString("kkk"); @@ -99,21 +102,21 @@ public class NbBundleTest extends TestCase { } catch (MissingResourceException mre) { // OK } - rb = NbBundle.getBundle("foo.Bundle", Locale.US, fixedLoader("foo/Bundle.properties:k=v")); + rb = NbBundle.getBundle("foo.Bundle", Locale.US, fixedLoader(false, "foo/Bundle.properties:k=v")); assertEquals("v", rb.getString("k")); - rb = NbBundle.getBundle("foo.Bundle", Locale.JAPAN, fixedLoader("foo/Bundle.properties:k=v")); + rb = NbBundle.getBundle("foo.Bundle", Locale.JAPAN, fixedLoader(false, "foo/Bundle.properties:k=v")); assertEquals("v", rb.getString("k")); - rb = NbBundle.getBundle("foo.Bundle", Locale.JAPAN, fixedLoader("foo/Bundle.properties:k=v", "foo/Bundle_ja.properties:k=v2")); + rb = NbBundle.getBundle("foo.Bundle", Locale.JAPAN, fixedLoader(false, "foo/Bundle.properties:k=v", "foo/Bundle_ja.properties:k=v2")); assertEquals("v2", rb.getString("k")); assertEquals(Locale.JAPAN, rb.getLocale()); try { - NbBundle.getBundle("foo.Bundle", Locale.ENGLISH, fixedLoader()); + NbBundle.getBundle("foo.Bundle", Locale.ENGLISH, fixedLoader(false)); fail(); } catch (MissingResourceException mre) { // OK } NbBundle.setBranding("nb"); - rb = NbBundle.getBundle("foo.Bundle", Locale.US, fixedLoader("foo/Bundle.properties:k1=v1\nk2=v2", "foo/Bundle_nb.properties:k1=v1 NB")); + rb = NbBundle.getBundle("foo.Bundle", Locale.US, fixedLoader(false, "foo/Bundle.properties:k1=v1\nk2=v2", "foo/Bundle_nb.properties:k1=v1 NB")); assertEquals("v1 NB", rb.getString("k1")); assertEquals("v2", rb.getString("k2")); List<String> keys = new ArrayList<String>(Collections.list(rb.getKeys())); @@ -121,14 +124,17 @@ public class NbBundleTest extends TestCase { assertEquals("[k1, k2]", keys.toString()); } - public void testGetMessage() throws Exception { - ClassLoader l = fixedLoader( - "org/openide/util/Bundle.properties:" + - "k1=v1\n" + - "k2=v2 {0}\n" + - "k3=v3 {0} {1} {2} {3} {4}", - "org/openide/util/Bundle_ja.properties:" + - "k1=v1 ja"); + public void testGetMessageISO() throws Exception { + ClassLoader l = fixedLoader(false, + "org/openide/util/Bundle.properties:" + + "k1=v1\n" + + "k2=v2 {0}\n" + + "k3=v3 {0} {1} {2} {3} {4}", + "org/openide/util/Bundle_ja.properties:" + + "k1=v1 ja", + "org/openide/util/Bundle_fr.properties:" + + "k1=v1 ô Hélène" + ); Class<?> c = l.loadClass(Dummy.class.getName()); assertEquals(l, c.getClassLoader()); assertEquals("v1", NbBundle.getMessage(c, "k1")); @@ -136,24 +142,87 @@ public class NbBundleTest extends TestCase { assertEquals("v1 ja", NbBundle.getMessage(c, "k1")); assertEquals("v2 x", NbBundle.getMessage(c, "k2", "x")); assertEquals("v3 a b c d e", NbBundle.getMessage(c, "k3", "a", "b", "c", "d", "e")); + Locale.setDefault(Locale.FRENCH); + assertEquals("v1 ô Hélène", NbBundle.getMessage(c, "k1")); + } + + public void testGetMessageUTF8() throws Exception { + ClassLoader l = fixedLoader(true, + "org/openide/util/Bundle.properties:" + + "k1=v1\n" + + "k2=v2 {0}\n" + + "k3=v3 {0} {1} {2} {3} {4}", + "org/openide/util/Bundle_ja.properties:" + + "k1=v1 ja", + "org/openide/util/Bundle_fr.properties:" + + "k2=v2 ô Hélène {0}" + ); + Class<?> c = l.loadClass(Dummy.class.getName()); + assertEquals(l, c.getClassLoader()); + assertEquals("v1", NbBundle.getMessage(c, "k1")); + Locale.setDefault(Locale.JAPAN); + assertEquals("v1 ja", NbBundle.getMessage(c, "k1")); + assertEquals("v2 x", NbBundle.getMessage(c, "k2", "x")); + assertEquals("v3 a b c d e", NbBundle.getMessage(c, "k3", "a", "b", "c", "d", "e")); + Locale.setDefault(Locale.FRENCH); + assertEquals("v2 ô Hélène chérie", NbBundle.getMessage(c, "k2", "chérie")); + } + + public void testSystemPropertyISO() throws Exception { + ClassLoader l = fixedLoader(true, + "org/openide/util/Bundle.properties:" + + "k1=yo\n" + + "k2=where and ouch", + "org/openide/util/Bundle_fr.properties:" + + "k1=fr yo\n" + + "k2=où et aïe" + ); + Class<?> c = l.loadClass(Dummy.class.getName()); + assertEquals(l, c.getClassLoader()); + Locale.setDefault(Locale.FRENCH); + System.setProperty("java.util.PropertyResourceBundle.encoding", "ISO-8859-1"); + assertEquals("fr yo", NbBundle.getMessage(c, "k1")); + assertNotEquals("où et aïe", NbBundle.getMessage(c, "k2")); + } + + public void testSystemPropertyUTF() throws Exception { + ClassLoader l = fixedLoader(false, + "org/openide/util/Bundle.properties:" + + "k1=yo\n" + + "k2=where and ouch", + "org/openide/util/Bundle_fr.properties:" + + "k1=fr yo\n" + + "k2=où et aïe" + ); + Class<?> c = l.loadClass(Dummy.class.getName()); + assertEquals(l, c.getClassLoader()); + Locale.setDefault(Locale.FRENCH); + System.setProperty("java.util.PropertyResourceBundle.encoding", "UTF-8"); + try { + assertEquals("fr yo", NbBundle.getMessage(c, "k1")); + fail(); + } catch (MissingResourceException mre) { + // OK MalformedInputException + } + } static class Dummy {} /** - * Creates a loader which can load just fixed resources you supply. - * Each entry should be of the form + * Creates a loader which can load just fixed resources you supply. Each + * entry should be of the form * <pre> * path/to/res1:some contents * for res1 - * </pre> - * Resources are expected to be in ISO-8859-1. - * Also can define a class named Dummy.class.getName(). + * </pre> Also can define a class named Dummy.class.getName(). + * + * @param useUTF8 If false use ISO-8859-1 encoding, otherwise UTF-8 */ - private static ClassLoader fixedLoader(String... entries) throws Exception { - final Map<String,byte[]> data = new HashMap<String,byte[]>(); + private static ClassLoader fixedLoader(boolean useUTF8, String... entries) throws Exception { + final Map<String, byte[]> data = new HashMap<String, byte[]>(); for (String entry : entries) { int colon = entry.indexOf(':'); - data.put(entry.substring(0, colon), entry.substring(colon + 1).getBytes("ISO-8859-1")); + data.put(entry.substring(0, colon), entry.substring(colon + 1).getBytes(useUTF8 ? "UTF-8" : "ISO-8859-1")); } return new ClassLoader() { @Override @@ -180,6 +249,7 @@ public class NbBundleTest extends TestCase { return null; } } + @Override public Class loadClass(String n) throws ClassNotFoundException { if (n.equals(Dummy.class.getName())) { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@netbeans.apache.org For additional commands, e-mail: commits-h...@netbeans.apache.org For further information about the NetBeans mailing lists, visit: https://cwiki.apache.org/confluence/display/NETBEANS/Mailing+lists