This is an automated email from the ASF dual-hosted git repository.
andy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/jena.git
The following commit(s) were added to refs/heads/master by this push:
new aba8e76 JENA-1793: Decode %-encoded UTF-8
new 2040407 Merge pull request #648 from afs/irilib-decode
aba8e76 is described below
commit aba8e762153f5cc436351b543e50453d0b55a88e
Author: Andy Seaborne <[email protected]>
AuthorDate: Mon Dec 9 10:45:50 2019 +0000
JENA-1793: Decode %-encoded UTF-8
---
.../src/main/java/org/apache/jena/atlas/io/IO.java | 4 +-
.../java/org/apache/jena/atlas/lib/IRILib.java | 25 ++++++++--
.../java/org/apache/jena/atlas/lib/StrUtils.java | 48 +++++++++++---------
.../java/org/apache/jena/atlas/lib/TS_Lib.java | 1 +
.../atlas/lib/{TS_Lib.java => TestIRILib.java} | 53 ++++++++--------------
5 files changed, 70 insertions(+), 61 deletions(-)
diff --git a/jena-base/src/main/java/org/apache/jena/atlas/io/IO.java
b/jena-base/src/main/java/org/apache/jena/atlas/io/IO.java
index d22f5c3..d01ddf2 100644
--- a/jena-base/src/main/java/org/apache/jena/atlas/io/IO.java
+++ b/jena-base/src/main/java/org/apache/jena/atlas/io/IO.java
@@ -85,7 +85,7 @@ public class IO
if ( filename.startsWith("file:") )
{
filename = filename.substring("file:".length());
- filename = IRILib.decode(filename);
+ filename = IRILib.decodeHex(filename);
}
InputStream in = new FileInputStream(filename);
String ext = FilenameUtils.getExtension(filename);
@@ -180,7 +180,7 @@ public class IO
if ( filename.startsWith("file:") )
{
filename = filename.substring("file:".length());
- filename = IRILib.decode(filename);
+ filename = IRILib.decodeHex(filename);
}
OutputStream out = new FileOutputStream(filename);
String ext = FilenameUtils.getExtension(filename);
diff --git a/jena-base/src/main/java/org/apache/jena/atlas/lib/IRILib.java
b/jena-base/src/main/java/org/apache/jena/atlas/lib/IRILib.java
index ffe05de..8c6dfff 100644
--- a/jena-base/src/main/java/org/apache/jena/atlas/lib/IRILib.java
+++ b/jena-base/src/main/java/org/apache/jena/atlas/lib/IRILib.java
@@ -232,10 +232,6 @@ public class IRILib
return uri ;
}
- public static String decode(String string) {
- return StrUtils.decodeHex(string, '%') ;
- }
-
public static String encodeNonASCII(String string) {
if ( ! containsNonASCII(string) )
return string ;
@@ -248,7 +244,7 @@ public class IRILib
sw.append( (char) b );
continue;
}
-
+
int hi = ( b & 0xF0 ) >> 4;
int lo = b & 0xF;
sw.append( '%' );
@@ -265,5 +261,24 @@ public class IRILib
return true;
}
return false ;
+ }
+
+ /** @deprecated Use {@link #decodeHex} */
+ @Deprecated
+ public static String decode(String string) { return decodeHex(string); }
+
+ /**
+ * Decode a string that may have %-encoded sequences.
+ * <p>
+ * This function will reverse
+ * {@link #encodeNonASCII(String)},
+ * {@link #encodeUriPath(String)},
+ * {@link #encodeFileURL(String)} and
+ * {@link #encodeUriComponent(String)}.
+ *
+ * It will not decode '+' used for space
(application/x-www-form-urlencoded).
+ */
+ public static String decodeHex(String string) {
+ return StrUtils.decodeHex(string, '%') ;
}
}
diff --git a/jena-base/src/main/java/org/apache/jena/atlas/lib/StrUtils.java
b/jena-base/src/main/java/org/apache/jena/atlas/lib/StrUtils.java
index f179bd3..796a7ca 100644
--- a/jena-base/src/main/java/org/apache/jena/atlas/lib/StrUtils.java
+++ b/jena-base/src/main/java/org/apache/jena/atlas/lib/StrUtils.java
@@ -22,6 +22,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Arrays.stream ;
import static java.util.stream.Collectors.toList;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -200,33 +201,38 @@ public class StrUtils //extends StringUtils
/**
* Decode a string using marked hex values e.g. %20
*
- * @param str String to decode
- * @param marker The marker charcater
+ * @param str String to decode : characters should be ASCII (<127)
+ * @param marker The marker character
* @return Decoded string (returns input object on no change)
*/
public static String decodeHex(String str, char marker) {
- int idx = str.indexOf(marker);
- if ( idx == -1 )
+ if ( str.indexOf(marker) < 0 )
return str;
- StringBuilder buff = new StringBuilder();
-
- buff.append(str, 0, idx);
- int N = str.length();
-
- for ( ; idx < N ; idx++ ) {
- char ch = str.charAt(idx);
- // First time through this is true, always.
- if ( ch != marker )
- buff.append(ch);
- else {
- char hi = str.charAt(idx + 1);
- char lo = str.charAt(idx + 2);
- char ch2 = (char)(hexDecode(hi) << 4 | hexDecode(lo));
- buff.append(ch2);
- idx += 2;
+ // This function does work if input str is not pure ASCII.
+ // The tricky part is if an %-encoded part is a UTF-8 sequence.
+ // An alternative algorithm is to work in chars from the string, and
handle
+ // that case %-endocded when value has the high bit set.
+ byte[] strBytes = StrUtils.asUTF8bytes(str);
+ final int N = strBytes.length;
+ // Max length
+ byte[] bytes = new byte[strBytes.length];
+ int i = 0;
+ for ( int j = 0 ; j < N ; j++ ) {
+ byte b = strBytes[j];
+ if ( b != marker ) {
+ bytes[i++] = b;
+ continue;
}
+ // Marker.
+ char hi = str.charAt(j + 1);
+ char lo = str.charAt(j + 2);
+ j += 2;
+ int x1 = hexDecode(hi);
+ int x2 = hexDecode(lo);
+ int ch2 = (hexDecode(hi) << 4 | hexDecode(lo));
+ bytes[i++] = (byte)ch2;
}
- return buff.toString();
+ return new String(bytes, 0, i, StandardCharsets.UTF_8);
}
// Encoding is table-driven but for decode, we use code.
diff --git a/jena-base/src/test/java/org/apache/jena/atlas/lib/TS_Lib.java
b/jena-base/src/test/java/org/apache/jena/atlas/lib/TS_Lib.java
index b05d47c..87abe6e 100644
--- a/jena-base/src/test/java/org/apache/jena/atlas/lib/TS_Lib.java
+++ b/jena-base/src/test/java/org/apache/jena/atlas/lib/TS_Lib.java
@@ -41,6 +41,7 @@ import org.junit.runners.Suite ;
, TestCache2.class
, TestFileOps.class
, TestStrUtils.class
+ , TestIRILib.class
, TestXMLLib.class
, TestAlarmClock.class
, TestTrie.class
diff --git a/jena-base/src/test/java/org/apache/jena/atlas/lib/TS_Lib.java
b/jena-base/src/test/java/org/apache/jena/atlas/lib/TestIRILib.java
similarity index 52%
copy from jena-base/src/test/java/org/apache/jena/atlas/lib/TS_Lib.java
copy to jena-base/src/test/java/org/apache/jena/atlas/lib/TestIRILib.java
index b05d47c..cd11e22 100644
--- a/jena-base/src/test/java/org/apache/jena/atlas/lib/TS_Lib.java
+++ b/jena-base/src/test/java/org/apache/jena/atlas/lib/TestIRILib.java
@@ -18,40 +18,27 @@
package org.apache.jena.atlas.lib;
+import static org.junit.Assert.assertEquals;
-import org.apache.jena.atlas.lib.cache.TestCacheSimple;
-import org.junit.runner.RunWith ;
-import org.junit.runners.Suite ;
+import org.junit.Test;
-/**
- * Tests for the Atlas lib package
- */
-@RunWith(Suite.class)
[email protected]( {
- TestAlg.class
- , TestBitsLong.class
- , TestBitsInt.class
- , TestBytes.class
- , TestEscapeStr.class
- , TestHex.class
- , TestListUtils.class
- , TestSetUtils.class
- , TestCollectionUtils.class
- , TestCache.class
- , TestCache2.class
- , TestFileOps.class
- , TestStrUtils.class
- , TestXMLLib.class
- , TestAlarmClock.class
- , TestTrie.class
- , TestFilenameProcessing.class
- , TestNumberUtils.class
- , TestDateTimeUtils.class
- , TestCacheSimple.class
- , TestRefCountingMap.class
-} )
-
-public class TS_Lib
-{
+public class TestIRILib {
+
+ @Test public void encodeDecode01() { encodeDecode(""); }
+
+ @Test public void encodeDecode02() { encodeDecode("aa"); }
+
+ @Test public void encodeDecode03() { encodeDecode("aa"); }
+
+ @Test public void encodeDecode04() { encodeDecode("Größe"); }
+
+ private void encodeDecode(String testString) {
+ String encoded = IRILib.encodeNonASCII(testString);
+ String decoded = IRILib.decodeHex(encoded);
+ if ( ! testString.equals(decoded) ) {
+ System.out.println(encoded);
+ }
+ assertEquals(testString, decoded);
+ }
}
