This is an automated email from the ASF dual-hosted git repository.

matthiasblaesing pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/netbeans.git


The following commit(s) were added to refs/heads/master by this push:
     new 3b8dea5  [NETBEANS-5181] NbBundle only accepts ISO-8859-1 while UTF-8 
is default since JDK9
     new 396d4be  Merge pull request #2633 from jjazzboss/NETBEANS-5181-NbBundle
3b8dea5 is described below

commit 3b8dea5f27a889a42f6356efd23a602418d6c5d3
Author: Jerome Lelasseux <j...@jjazzlab.com>
AuthorDate: Fri Jan 1 21:20:47 2021 +0100

    [NETBEANS-5181] NbBundle only accepts ISO-8859-1 while UTF-8 is default 
since JDK9
    
    Change NbBundle to read Bundle files using UTF-1 encoding by default, and
    automatically switch to ISO-8859-1 if UTF-1 decoding fails. System property
    java.util.PropertyResourceBundle.encoding can can alter the behavior
    like for PropertyResourceBundle (from JDK9).
---
 .../src/org/openide/util/NbBundle.java             | 106 +++++++++++++++++-
 .../unit/src/org/openide/util/NbBundleTest.java    | 120 ++++++++++++++++-----
 2 files changed, 199 insertions(+), 27 deletions(-)

diff --git a/platform/openide.util/src/org/openide/util/NbBundle.java 
b/platform/openide.util/src/org/openide/util/NbBundle.java
index 77b3473..e072141 100644
--- a/platform/openide.util/src/org/openide/util/NbBundle.java
+++ b/platform/openide.util/src/org/openide/util/NbBundle.java
@@ -21,6 +21,7 @@ package org.openide.util;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
@@ -28,6 +29,14 @@ import java.lang.annotation.Target;
 import java.lang.ref.Reference;
 import java.lang.ref.WeakReference;
 import java.net.URL;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CoderResult;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Enumeration;
@@ -68,7 +77,10 @@ public class NbBundle extends Object {
 
     private static final boolean USE_DEBUG_LOADER = 
Boolean.getBoolean("org.openide.util.NbBundle.DEBUG"); // NOI18N
     private static String brandingToken = null;
-
+    
+    private static final UtfThenIsoCharset utfThenIsoCharset = new 
UtfThenIsoCharset(false);
+    private static final UtfThenIsoCharset utfThenIsoCharsetOnlyUTF8 = new 
UtfThenIsoCharset(true);    
+    
     /**
      * Cache of URLs for localized files.
      * Keeps only weak references to the class loaders.
@@ -538,8 +550,16 @@ public class NbBundle extends Object {
                         (loader != null ? loader.getResourceAsStream(res) : 
ClassLoader.getSystemResourceAsStream(res)) :
                             u.openStream();
 
+                    // #NETBEANS-5181
+                    String encoding = 
System.getProperty("java.util.PropertyResourceBundle.encoding");
+                    UtfThenIsoCharset charset = "UTF-8".equals(encoding) ? 
utfThenIsoCharsetOnlyUTF8 : utfThenIsoCharset;
+                    InputStreamReader reader = new InputStreamReader(is,
+                            "ISO-8859-1".equals(encoding)
+                            ? StandardCharsets.ISO_8859_1.newDecoder()
+                            : charset.newDecoder());
+
                     try {
-                        p.load(is);
+                        p.load(reader);
                     } finally {
                         is.close();
                     }
@@ -1458,4 +1478,86 @@ public class NbBundle extends Object {
 
         }
     }
+    
+    
+    /**
+     * Local charset to decode using UTF-8 by default, but automatically 
switching to ISO-8859-1 if UTF-8 decoding fails.
+     * 
+     */
+    static private class UtfThenIsoCharset extends Charset {
+
+        private final boolean onlyUTF8;
+
+        /**
+         *
+         * @param acceptOnlyUTF8 If true there is no automatic switch to 
ISO-8859-1 if UTF-8 decoding fails.
+         */
+        public UtfThenIsoCharset(boolean acceptOnlyUTF8) {
+            super(UtfThenIsoCharset.class.getCanonicalName(), null);
+            this.onlyUTF8 = acceptOnlyUTF8;
+        }
+
+        @Override
+        public boolean contains(Charset arg0) {
+            return this.equals(arg0);
+        }
+
+        @Override
+        public CharsetDecoder newDecoder() {
+            return new UtfThenIsoDecoder(this, 1.0f, 1.0f);
+        }
+
+        @Override
+        public CharsetEncoder newEncoder() {
+            throw new UnsupportedOperationException("Not supported yet.");
+        }
+
+         private final class UtfThenIsoDecoder extends CharsetDecoder {
+
+            private CharsetDecoder decoderUTF;
+            private CharsetDecoder decoderISO;  // Not null means we switched 
to ISO
+
+            protected UtfThenIsoDecoder(Charset cs, float averageCharsPerByte, 
float maxCharsPerByte) {
+                super(cs, averageCharsPerByte, maxCharsPerByte);
+
+                decoderUTF = StandardCharsets.UTF_8.newDecoder()
+                        .onMalformedInput(CodingErrorAction.REPORT) // We want 
to be informed of this error
+                        .onUnmappableCharacter(CodingErrorAction.REPORT);  // 
We want to be informed of this error                
+            }
+
+            @Override
+            protected CoderResult decodeLoop(ByteBuffer in, CharBuffer out) {
+
+                if (decoderISO != null) {
+                    // No turning back once we've switched to ISO
+                    return decoderISO.decode(in, out, false);
+                }
+
+                // To rewind if need to retry with ISO decoding
+                in.mark();
+                out.mark();
+
+                
+                // UTF decoding
+                CoderResult cr = decoderUTF.decode(in, out, false);
+                if (cr.isUnderflow() || cr.isOverflow()) {
+                    // Normal results
+                    return cr;
+                }
+
+                // If we're here there was a malformed-input or 
unmappable-character error with the UTF decoding
+                if (UtfThenIsoCharset.this.onlyUTF8) {
+                    // But can't switch to ISO
+                    return cr;
+                }
+
+                // Switch to ISO
+                in.reset();
+                out.reset();
+                decoderISO = StandardCharsets.ISO_8859_1.newDecoder();
+                return decoderISO.decode(in, out, false);
+            }
+        }
+    }
+    
 }
diff --git 
a/platform/openide.util/test/unit/src/org/openide/util/NbBundleTest.java 
b/platform/openide.util/test/unit/src/org/openide/util/NbBundleTest.java
index fce554a..444e042 100644
--- a/platform/openide.util/test/unit/src/org/openide/util/NbBundleTest.java
+++ b/platform/openide.util/test/unit/src/org/openide/util/NbBundleTest.java
@@ -16,7 +16,6 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
 package org.openide.util;
 
 import java.io.ByteArrayInputStream;
@@ -40,8 +39,10 @@ import java.util.ResourceBundle;
 import java.util.TreeMap;
 import java.util.jar.Attributes;
 import junit.framework.TestCase;
-import org.openide.util.NbBundle;
-import org.openide.util.NbCollections;
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertNotNull;
+import static junit.framework.TestCase.fail;
+import static org.junit.Assert.assertNotEquals;
 import org.openide.util.base.BundleClass;
 
 // XXX testGetClassBundle
@@ -63,6 +64,7 @@ public class NbBundleTest extends TestCase {
         NbBundle.setBranding(null);
         NbBundle.localizedFileCache.clear();
         NbBundle.bundleCache.clear();
+        System.clearProperty("java.util.PropertyResourceBundle.encoding");
     }
 
     public static void testLocalizingSuffixes() throws Exception {
@@ -75,6 +77,7 @@ public class NbBundleTest extends TestCase {
         Locale.setDefault(Locale.JAPAN);
         
assertEquals("_f4j_ce_ja_JP,_f4j_ce_ja,_f4j_ce,_f4j_ja_JP,_f4j_ja,_f4j,_ja_JP,_ja,",
 locSuff());
     }
+
     private static String locSuff() {
         StringBuffer b = new StringBuffer();
         boolean first = true;
@@ -91,7 +94,7 @@ public class NbBundleTest extends TestCase {
     }
 
     public void testGetBundle() throws Exception {
-        ResourceBundle rb = NbBundle.getBundle("foo.Bundle", Locale.ENGLISH, 
fixedLoader("foo/Bundle.properties:k=v"));
+        ResourceBundle rb = NbBundle.getBundle("foo.Bundle", Locale.ENGLISH, 
fixedLoader(false, "foo/Bundle.properties:k=v"));
         assertEquals("v", rb.getString("k"));
         try {
             rb.getString("kkk");
@@ -99,21 +102,21 @@ public class NbBundleTest extends TestCase {
         } catch (MissingResourceException mre) {
             // OK
         }
-        rb = NbBundle.getBundle("foo.Bundle", Locale.US, 
fixedLoader("foo/Bundle.properties:k=v"));
+        rb = NbBundle.getBundle("foo.Bundle", Locale.US, fixedLoader(false, 
"foo/Bundle.properties:k=v"));
         assertEquals("v", rb.getString("k"));
-        rb = NbBundle.getBundle("foo.Bundle", Locale.JAPAN, 
fixedLoader("foo/Bundle.properties:k=v"));
+        rb = NbBundle.getBundle("foo.Bundle", Locale.JAPAN, fixedLoader(false, 
"foo/Bundle.properties:k=v"));
         assertEquals("v", rb.getString("k"));
-        rb = NbBundle.getBundle("foo.Bundle", Locale.JAPAN, 
fixedLoader("foo/Bundle.properties:k=v", "foo/Bundle_ja.properties:k=v2"));
+        rb = NbBundle.getBundle("foo.Bundle", Locale.JAPAN, fixedLoader(false, 
"foo/Bundle.properties:k=v", "foo/Bundle_ja.properties:k=v2"));
         assertEquals("v2", rb.getString("k"));
         assertEquals(Locale.JAPAN, rb.getLocale());
         try {
-            NbBundle.getBundle("foo.Bundle", Locale.ENGLISH, fixedLoader());
+            NbBundle.getBundle("foo.Bundle", Locale.ENGLISH, 
fixedLoader(false));
             fail();
         } catch (MissingResourceException mre) {
             // OK
         }
         NbBundle.setBranding("nb");
-        rb = NbBundle.getBundle("foo.Bundle", Locale.US, 
fixedLoader("foo/Bundle.properties:k1=v1\nk2=v2", 
"foo/Bundle_nb.properties:k1=v1 NB"));
+        rb = NbBundle.getBundle("foo.Bundle", Locale.US, fixedLoader(false, 
"foo/Bundle.properties:k1=v1\nk2=v2", "foo/Bundle_nb.properties:k1=v1 NB"));
         assertEquals("v1 NB", rb.getString("k1"));
         assertEquals("v2", rb.getString("k2"));
         List<String> keys = new 
ArrayList<String>(Collections.list(rb.getKeys()));
@@ -121,14 +124,17 @@ public class NbBundleTest extends TestCase {
         assertEquals("[k1, k2]", keys.toString());
     }
 
-    public void testGetMessage() throws Exception {
-        ClassLoader l = fixedLoader(
-                "org/openide/util/Bundle.properties:" +
-                "k1=v1\n" +
-                "k2=v2 {0}\n" +
-                "k3=v3 {0} {1} {2} {3} {4}",
-                "org/openide/util/Bundle_ja.properties:" +
-                "k1=v1 ja");
+    public void testGetMessageISO() throws Exception {
+        ClassLoader l = fixedLoader(false,
+                "org/openide/util/Bundle.properties:"
+                + "k1=v1\n"
+                + "k2=v2 {0}\n"
+                + "k3=v3 {0} {1} {2} {3} {4}",
+                "org/openide/util/Bundle_ja.properties:"
+                + "k1=v1 ja",
+                "org/openide/util/Bundle_fr.properties:"
+                + "k1=v1 ô Hélène"
+        );
         Class<?> c = l.loadClass(Dummy.class.getName());
         assertEquals(l, c.getClassLoader());
         assertEquals("v1", NbBundle.getMessage(c, "k1"));
@@ -136,24 +142,87 @@ public class NbBundleTest extends TestCase {
         assertEquals("v1 ja", NbBundle.getMessage(c, "k1"));
         assertEquals("v2 x", NbBundle.getMessage(c, "k2", "x"));
         assertEquals("v3 a b c d e", NbBundle.getMessage(c, "k3", "a", "b", 
"c", "d", "e"));
+        Locale.setDefault(Locale.FRENCH);
+        assertEquals("v1 ô Hélène", NbBundle.getMessage(c, "k1"));
+    }
+
+    public void testGetMessageUTF8() throws Exception {
+        ClassLoader l = fixedLoader(true,
+                "org/openide/util/Bundle.properties:"
+                + "k1=v1\n"
+                + "k2=v2 {0}\n"
+                + "k3=v3 {0} {1} {2} {3} {4}",
+                "org/openide/util/Bundle_ja.properties:"
+                + "k1=v1 ja",
+                "org/openide/util/Bundle_fr.properties:"
+                + "k2=v2 ô Hélène {0}"
+        );
+        Class<?> c = l.loadClass(Dummy.class.getName());
+        assertEquals(l, c.getClassLoader());
+        assertEquals("v1", NbBundle.getMessage(c, "k1"));
+        Locale.setDefault(Locale.JAPAN);
+        assertEquals("v1 ja", NbBundle.getMessage(c, "k1"));
+        assertEquals("v2 x", NbBundle.getMessage(c, "k2", "x"));
+        assertEquals("v3 a b c d e", NbBundle.getMessage(c, "k3", "a", "b", 
"c", "d", "e"));
+        Locale.setDefault(Locale.FRENCH);
+        assertEquals("v2 ô Hélène chérie", NbBundle.getMessage(c, "k2", 
"chérie"));
+    }
+
+    public void testSystemPropertyISO() throws Exception {
+        ClassLoader l = fixedLoader(true,
+                "org/openide/util/Bundle.properties:"
+                + "k1=yo\n"
+                + "k2=where and ouch",
+                "org/openide/util/Bundle_fr.properties:"
+                + "k1=fr yo\n"
+                + "k2=où et aïe"
+        );
+        Class<?> c = l.loadClass(Dummy.class.getName());
+        assertEquals(l, c.getClassLoader());
+        Locale.setDefault(Locale.FRENCH);
+        System.setProperty("java.util.PropertyResourceBundle.encoding", 
"ISO-8859-1");
+        assertEquals("fr yo", NbBundle.getMessage(c, "k1"));
+        assertNotEquals("où et aïe", NbBundle.getMessage(c, "k2"));
+    }
+
+    public void testSystemPropertyUTF() throws Exception {
+        ClassLoader l = fixedLoader(false,
+                "org/openide/util/Bundle.properties:"
+                + "k1=yo\n"
+                + "k2=where and ouch",
+                "org/openide/util/Bundle_fr.properties:"
+                + "k1=fr yo\n"
+                + "k2=où et aïe"
+        );
+        Class<?> c = l.loadClass(Dummy.class.getName());
+        assertEquals(l, c.getClassLoader());
+        Locale.setDefault(Locale.FRENCH);
+        System.setProperty("java.util.PropertyResourceBundle.encoding", 
"UTF-8");
+        try {
+            assertEquals("fr yo", NbBundle.getMessage(c, "k1"));
+            fail();
+        } catch (MissingResourceException mre) {
+            // OK MalformedInputException
+        }
+
     }
 
     static class Dummy {}
     /**
-     * Creates a loader which can load just fixed resources you supply.
-     * Each entry should be of the form
+     * Creates a loader which can load just fixed resources you supply. Each
+     * entry should be of the form
      * <pre>
      * path/to/res1:some contents
      * for res1
-     * </pre>
-     * Resources are expected to be in ISO-8859-1.
-     * Also can define a class named Dummy.class.getName().
+     * </pre> Also can define a class named Dummy.class.getName().
+     *
+     * @param useUTF8 If false use ISO-8859-1 encoding, otherwise UTF-8
      */
-    private static ClassLoader fixedLoader(String... entries) throws Exception 
{
-        final Map<String,byte[]> data = new HashMap<String,byte[]>();
+    private static ClassLoader fixedLoader(boolean useUTF8, String... entries) 
throws Exception {
+        final Map<String, byte[]> data = new HashMap<String, byte[]>();
         for (String entry : entries) {
             int colon = entry.indexOf(':');
-            data.put(entry.substring(0, colon), entry.substring(colon + 
1).getBytes("ISO-8859-1"));
+            data.put(entry.substring(0, colon), entry.substring(colon + 
1).getBytes(useUTF8 ? "UTF-8" : "ISO-8859-1"));
         }
         return new ClassLoader() {
             @Override
@@ -180,6 +249,7 @@ public class NbBundleTest extends TestCase {
                     return null;
                 }
             }
+
             @Override
             public Class loadClass(String n) throws ClassNotFoundException {
                 if (n.equals(Dummy.class.getName())) {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@netbeans.apache.org
For additional commands, e-mail: commits-h...@netbeans.apache.org

For further information about the NetBeans mailing lists, visit:
https://cwiki.apache.org/confluence/display/NETBEANS/Mailing+lists

Reply via email to