This is an automated email from the ASF dual-hosted git repository.
matthiasblaesing pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/netbeans.git
The following commit(s) were added to refs/heads/master by this push:
new 3b8dea5 [NETBEANS-5181] NbBundle only accepts ISO-8859-1 while UTF-8
is default since JDK9
new 396d4be Merge pull request #2633 from jjazzboss/NETBEANS-5181-NbBundle
3b8dea5 is described below
commit 3b8dea5f27a889a42f6356efd23a602418d6c5d3
Author: Jerome Lelasseux <[email protected]>
AuthorDate: Fri Jan 1 21:20:47 2021 +0100
[NETBEANS-5181] NbBundle only accepts ISO-8859-1 while UTF-8 is default
since JDK9
Change NbBundle to read Bundle files using UTF-1 encoding by default, and
automatically switch to ISO-8859-1 if UTF-1 decoding fails. System property
java.util.PropertyResourceBundle.encoding can can alter the behavior
like for PropertyResourceBundle (from JDK9).
---
.../src/org/openide/util/NbBundle.java | 106 +++++++++++++++++-
.../unit/src/org/openide/util/NbBundleTest.java | 120 ++++++++++++++++-----
2 files changed, 199 insertions(+), 27 deletions(-)
diff --git a/platform/openide.util/src/org/openide/util/NbBundle.java
b/platform/openide.util/src/org/openide/util/NbBundle.java
index 77b3473..e072141 100644
--- a/platform/openide.util/src/org/openide/util/NbBundle.java
+++ b/platform/openide.util/src/org/openide/util/NbBundle.java
@@ -21,6 +21,7 @@ package org.openide.util;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@@ -28,6 +29,14 @@ import java.lang.annotation.Target;
import java.lang.ref.Reference;
import java.lang.ref.WeakReference;
import java.net.URL;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CoderResult;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;
@@ -68,7 +77,10 @@ public class NbBundle extends Object {
private static final boolean USE_DEBUG_LOADER =
Boolean.getBoolean("org.openide.util.NbBundle.DEBUG"); // NOI18N
private static String brandingToken = null;
-
+
+ private static final UtfThenIsoCharset utfThenIsoCharset = new
UtfThenIsoCharset(false);
+ private static final UtfThenIsoCharset utfThenIsoCharsetOnlyUTF8 = new
UtfThenIsoCharset(true);
+
/**
* Cache of URLs for localized files.
* Keeps only weak references to the class loaders.
@@ -538,8 +550,16 @@ public class NbBundle extends Object {
(loader != null ? loader.getResourceAsStream(res) :
ClassLoader.getSystemResourceAsStream(res)) :
u.openStream();
+ // #NETBEANS-5181
+ String encoding =
System.getProperty("java.util.PropertyResourceBundle.encoding");
+ UtfThenIsoCharset charset = "UTF-8".equals(encoding) ?
utfThenIsoCharsetOnlyUTF8 : utfThenIsoCharset;
+ InputStreamReader reader = new InputStreamReader(is,
+ "ISO-8859-1".equals(encoding)
+ ? StandardCharsets.ISO_8859_1.newDecoder()
+ : charset.newDecoder());
+
try {
- p.load(is);
+ p.load(reader);
} finally {
is.close();
}
@@ -1458,4 +1478,86 @@ public class NbBundle extends Object {
}
}
+
+
+ /**
+ * Local charset to decode using UTF-8 by default, but automatically
switching to ISO-8859-1 if UTF-8 decoding fails.
+ *
+ */
+ static private class UtfThenIsoCharset extends Charset {
+
+ private final boolean onlyUTF8;
+
+ /**
+ *
+ * @param acceptOnlyUTF8 If true there is no automatic switch to
ISO-8859-1 if UTF-8 decoding fails.
+ */
+ public UtfThenIsoCharset(boolean acceptOnlyUTF8) {
+ super(UtfThenIsoCharset.class.getCanonicalName(), null);
+ this.onlyUTF8 = acceptOnlyUTF8;
+ }
+
+ @Override
+ public boolean contains(Charset arg0) {
+ return this.equals(arg0);
+ }
+
+ @Override
+ public CharsetDecoder newDecoder() {
+ return new UtfThenIsoDecoder(this, 1.0f, 1.0f);
+ }
+
+ @Override
+ public CharsetEncoder newEncoder() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ private final class UtfThenIsoDecoder extends CharsetDecoder {
+
+ private CharsetDecoder decoderUTF;
+ private CharsetDecoder decoderISO; // Not null means we switched
to ISO
+
+ protected UtfThenIsoDecoder(Charset cs, float averageCharsPerByte,
float maxCharsPerByte) {
+ super(cs, averageCharsPerByte, maxCharsPerByte);
+
+ decoderUTF = StandardCharsets.UTF_8.newDecoder()
+ .onMalformedInput(CodingErrorAction.REPORT) // We want
to be informed of this error
+ .onUnmappableCharacter(CodingErrorAction.REPORT); //
We want to be informed of this error
+ }
+
+ @Override
+ protected CoderResult decodeLoop(ByteBuffer in, CharBuffer out) {
+
+ if (decoderISO != null) {
+ // No turning back once we've switched to ISO
+ return decoderISO.decode(in, out, false);
+ }
+
+ // To rewind if need to retry with ISO decoding
+ in.mark();
+ out.mark();
+
+
+ // UTF decoding
+ CoderResult cr = decoderUTF.decode(in, out, false);
+ if (cr.isUnderflow() || cr.isOverflow()) {
+ // Normal results
+ return cr;
+ }
+
+ // If we're here there was a malformed-input or
unmappable-character error with the UTF decoding
+ if (UtfThenIsoCharset.this.onlyUTF8) {
+ // But can't switch to ISO
+ return cr;
+ }
+
+ // Switch to ISO
+ in.reset();
+ out.reset();
+ decoderISO = StandardCharsets.ISO_8859_1.newDecoder();
+ return decoderISO.decode(in, out, false);
+ }
+ }
+ }
+
}
diff --git
a/platform/openide.util/test/unit/src/org/openide/util/NbBundleTest.java
b/platform/openide.util/test/unit/src/org/openide/util/NbBundleTest.java
index fce554a..444e042 100644
--- a/platform/openide.util/test/unit/src/org/openide/util/NbBundleTest.java
+++ b/platform/openide.util/test/unit/src/org/openide/util/NbBundleTest.java
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.openide.util;
import java.io.ByteArrayInputStream;
@@ -40,8 +39,10 @@ import java.util.ResourceBundle;
import java.util.TreeMap;
import java.util.jar.Attributes;
import junit.framework.TestCase;
-import org.openide.util.NbBundle;
-import org.openide.util.NbCollections;
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertNotNull;
+import static junit.framework.TestCase.fail;
+import static org.junit.Assert.assertNotEquals;
import org.openide.util.base.BundleClass;
// XXX testGetClassBundle
@@ -63,6 +64,7 @@ public class NbBundleTest extends TestCase {
NbBundle.setBranding(null);
NbBundle.localizedFileCache.clear();
NbBundle.bundleCache.clear();
+ System.clearProperty("java.util.PropertyResourceBundle.encoding");
}
public static void testLocalizingSuffixes() throws Exception {
@@ -75,6 +77,7 @@ public class NbBundleTest extends TestCase {
Locale.setDefault(Locale.JAPAN);
assertEquals("_f4j_ce_ja_JP,_f4j_ce_ja,_f4j_ce,_f4j_ja_JP,_f4j_ja,_f4j,_ja_JP,_ja,",
locSuff());
}
+
private static String locSuff() {
StringBuffer b = new StringBuffer();
boolean first = true;
@@ -91,7 +94,7 @@ public class NbBundleTest extends TestCase {
}
public void testGetBundle() throws Exception {
- ResourceBundle rb = NbBundle.getBundle("foo.Bundle", Locale.ENGLISH,
fixedLoader("foo/Bundle.properties:k=v"));
+ ResourceBundle rb = NbBundle.getBundle("foo.Bundle", Locale.ENGLISH,
fixedLoader(false, "foo/Bundle.properties:k=v"));
assertEquals("v", rb.getString("k"));
try {
rb.getString("kkk");
@@ -99,21 +102,21 @@ public class NbBundleTest extends TestCase {
} catch (MissingResourceException mre) {
// OK
}
- rb = NbBundle.getBundle("foo.Bundle", Locale.US,
fixedLoader("foo/Bundle.properties:k=v"));
+ rb = NbBundle.getBundle("foo.Bundle", Locale.US, fixedLoader(false,
"foo/Bundle.properties:k=v"));
assertEquals("v", rb.getString("k"));
- rb = NbBundle.getBundle("foo.Bundle", Locale.JAPAN,
fixedLoader("foo/Bundle.properties:k=v"));
+ rb = NbBundle.getBundle("foo.Bundle", Locale.JAPAN, fixedLoader(false,
"foo/Bundle.properties:k=v"));
assertEquals("v", rb.getString("k"));
- rb = NbBundle.getBundle("foo.Bundle", Locale.JAPAN,
fixedLoader("foo/Bundle.properties:k=v", "foo/Bundle_ja.properties:k=v2"));
+ rb = NbBundle.getBundle("foo.Bundle", Locale.JAPAN, fixedLoader(false,
"foo/Bundle.properties:k=v", "foo/Bundle_ja.properties:k=v2"));
assertEquals("v2", rb.getString("k"));
assertEquals(Locale.JAPAN, rb.getLocale());
try {
- NbBundle.getBundle("foo.Bundle", Locale.ENGLISH, fixedLoader());
+ NbBundle.getBundle("foo.Bundle", Locale.ENGLISH,
fixedLoader(false));
fail();
} catch (MissingResourceException mre) {
// OK
}
NbBundle.setBranding("nb");
- rb = NbBundle.getBundle("foo.Bundle", Locale.US,
fixedLoader("foo/Bundle.properties:k1=v1\nk2=v2",
"foo/Bundle_nb.properties:k1=v1 NB"));
+ rb = NbBundle.getBundle("foo.Bundle", Locale.US, fixedLoader(false,
"foo/Bundle.properties:k1=v1\nk2=v2", "foo/Bundle_nb.properties:k1=v1 NB"));
assertEquals("v1 NB", rb.getString("k1"));
assertEquals("v2", rb.getString("k2"));
List<String> keys = new
ArrayList<String>(Collections.list(rb.getKeys()));
@@ -121,14 +124,17 @@ public class NbBundleTest extends TestCase {
assertEquals("[k1, k2]", keys.toString());
}
- public void testGetMessage() throws Exception {
- ClassLoader l = fixedLoader(
- "org/openide/util/Bundle.properties:" +
- "k1=v1\n" +
- "k2=v2 {0}\n" +
- "k3=v3 {0} {1} {2} {3} {4}",
- "org/openide/util/Bundle_ja.properties:" +
- "k1=v1 ja");
+ public void testGetMessageISO() throws Exception {
+ ClassLoader l = fixedLoader(false,
+ "org/openide/util/Bundle.properties:"
+ + "k1=v1\n"
+ + "k2=v2 {0}\n"
+ + "k3=v3 {0} {1} {2} {3} {4}",
+ "org/openide/util/Bundle_ja.properties:"
+ + "k1=v1 ja",
+ "org/openide/util/Bundle_fr.properties:"
+ + "k1=v1 ô Hélène"
+ );
Class<?> c = l.loadClass(Dummy.class.getName());
assertEquals(l, c.getClassLoader());
assertEquals("v1", NbBundle.getMessage(c, "k1"));
@@ -136,24 +142,87 @@ public class NbBundleTest extends TestCase {
assertEquals("v1 ja", NbBundle.getMessage(c, "k1"));
assertEquals("v2 x", NbBundle.getMessage(c, "k2", "x"));
assertEquals("v3 a b c d e", NbBundle.getMessage(c, "k3", "a", "b",
"c", "d", "e"));
+ Locale.setDefault(Locale.FRENCH);
+ assertEquals("v1 ô Hélène", NbBundle.getMessage(c, "k1"));
+ }
+
+ public void testGetMessageUTF8() throws Exception {
+ ClassLoader l = fixedLoader(true,
+ "org/openide/util/Bundle.properties:"
+ + "k1=v1\n"
+ + "k2=v2 {0}\n"
+ + "k3=v3 {0} {1} {2} {3} {4}",
+ "org/openide/util/Bundle_ja.properties:"
+ + "k1=v1 ja",
+ "org/openide/util/Bundle_fr.properties:"
+ + "k2=v2 ô Hélène {0}"
+ );
+ Class<?> c = l.loadClass(Dummy.class.getName());
+ assertEquals(l, c.getClassLoader());
+ assertEquals("v1", NbBundle.getMessage(c, "k1"));
+ Locale.setDefault(Locale.JAPAN);
+ assertEquals("v1 ja", NbBundle.getMessage(c, "k1"));
+ assertEquals("v2 x", NbBundle.getMessage(c, "k2", "x"));
+ assertEquals("v3 a b c d e", NbBundle.getMessage(c, "k3", "a", "b",
"c", "d", "e"));
+ Locale.setDefault(Locale.FRENCH);
+ assertEquals("v2 ô Hélène chérie", NbBundle.getMessage(c, "k2",
"chérie"));
+ }
+
+ public void testSystemPropertyISO() throws Exception {
+ ClassLoader l = fixedLoader(true,
+ "org/openide/util/Bundle.properties:"
+ + "k1=yo\n"
+ + "k2=where and ouch",
+ "org/openide/util/Bundle_fr.properties:"
+ + "k1=fr yo\n"
+ + "k2=où et aïe"
+ );
+ Class<?> c = l.loadClass(Dummy.class.getName());
+ assertEquals(l, c.getClassLoader());
+ Locale.setDefault(Locale.FRENCH);
+ System.setProperty("java.util.PropertyResourceBundle.encoding",
"ISO-8859-1");
+ assertEquals("fr yo", NbBundle.getMessage(c, "k1"));
+ assertNotEquals("où et aïe", NbBundle.getMessage(c, "k2"));
+ }
+
+ public void testSystemPropertyUTF() throws Exception {
+ ClassLoader l = fixedLoader(false,
+ "org/openide/util/Bundle.properties:"
+ + "k1=yo\n"
+ + "k2=where and ouch",
+ "org/openide/util/Bundle_fr.properties:"
+ + "k1=fr yo\n"
+ + "k2=où et aïe"
+ );
+ Class<?> c = l.loadClass(Dummy.class.getName());
+ assertEquals(l, c.getClassLoader());
+ Locale.setDefault(Locale.FRENCH);
+ System.setProperty("java.util.PropertyResourceBundle.encoding",
"UTF-8");
+ try {
+ assertEquals("fr yo", NbBundle.getMessage(c, "k1"));
+ fail();
+ } catch (MissingResourceException mre) {
+ // OK MalformedInputException
+ }
+
}
static class Dummy {}
/**
- * Creates a loader which can load just fixed resources you supply.
- * Each entry should be of the form
+ * Creates a loader which can load just fixed resources you supply. Each
+ * entry should be of the form
* <pre>
* path/to/res1:some contents
* for res1
- * </pre>
- * Resources are expected to be in ISO-8859-1.
- * Also can define a class named Dummy.class.getName().
+ * </pre> Also can define a class named Dummy.class.getName().
+ *
+ * @param useUTF8 If false use ISO-8859-1 encoding, otherwise UTF-8
*/
- private static ClassLoader fixedLoader(String... entries) throws Exception
{
- final Map<String,byte[]> data = new HashMap<String,byte[]>();
+ private static ClassLoader fixedLoader(boolean useUTF8, String... entries)
throws Exception {
+ final Map<String, byte[]> data = new HashMap<String, byte[]>();
for (String entry : entries) {
int colon = entry.indexOf(':');
- data.put(entry.substring(0, colon), entry.substring(colon +
1).getBytes("ISO-8859-1"));
+ data.put(entry.substring(0, colon), entry.substring(colon +
1).getBytes(useUTF8 ? "UTF-8" : "ISO-8859-1"));
}
return new ClassLoader() {
@Override
@@ -180,6 +249,7 @@ public class NbBundleTest extends TestCase {
return null;
}
}
+
@Override
public Class loadClass(String n) throws ClassNotFoundException {
if (n.equals(Dummy.class.getName())) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
For further information about the NetBeans mailing lists, visit:
https://cwiki.apache.org/confluence/display/NETBEANS/Mailing+lists
