This is an automated email from the ASF dual-hosted git repository.
pjfanning pushed a commit to branch 6.0.x
in repository https://gitbox.apache.org/repos/asf/poi.git
The following commit(s) were added to refs/heads/6.0.x by this push:
new dd9fc4711b [6.0.x] CodePageUtil / VBAMacroReader charset fixes (#1527)
dd9fc4711b is described below
commit dd9fc4711b735c204a152e88b540b90a068e1961
Author: PJ Fanning <[email protected]>
AuthorDate: Tue Nov 3 20:53:24 2026 +0100
[6.0.x] CodePageUtil / VBAMacroReader charset fixes (#1527)
* CodePageUtil: return Cp1256 for CP_WINDOWS_1256 when javaLangFormat is
true (#1517)
CP_WINDOWS_1256 (1256, Arabic) returned "Cp1255" (Hebrew) in the
javaLangFormat branch while returning "windows-1256" in the NIO branch.
Closes #1513
Generated-by: Google Gemini
(cherry picked from commit 71e95654e855402d0b186de7077a44cd565f5596)
* CodePageUtil: map Windows code pages 932 and 950 to Microsoft charsets
(#1518)
* CodePageUtil: map Windows code pages 932 and 950 to Microsoft charsets
Map CP_SJIS (932) to MS932 / windows-31j instead of strict JIS X 0208
Shift_JIS, and map code page 950 to MS950 / x-windows-950 instead of
falling through to "cp950" (which resolves to IBM's x-IBM950 in Java).
CP_MAC_JAPAN (10001) continues to return "SJIS".
Closes #1514
Generated-by: Google Gemini
* Update HSMF TestBasics to expect MS950 for code page 950
CodePageUtil.codepageToEncoding(950, true) now returns MS950
(Microsoft's Big5 variant) instead of cp950, so the encoding guessed
for chinese-traditional.msg changes accordingly.
(cherry picked from commit e7545612b9817dd36d4f53babb32b5b952ffd993)
* VBAMacroReader: key module map by StreamNameUnicode (#1516)
Per [MS-OVBA] 2.3.4.2.3.2.3 and [MS-CFB] 2.6.1, StreamNameUnicode holds
the UTF-16 stream name matching the compound file directory entry name.
Using pair.unicode() (falling back to pair.ascii() only when empty)
avoids a NullPointerException in readMacros() and preserves module type
lookup via PROJECTwm when the MBCS StreamName does not round-trip
through the project charset.
Closes #1512
Generated-by: Google Gemini
(cherry picked from commit 901eda24c7e965a64b1819b8a6603dd9d96287ad)
* VBAMacroReader: fall back to windows-1252 for unsupported project code
pages (#1519)
When PROJECT_CODEPAGE specifies a code page that CodePageUtil or the JVM
cannot resolve, log a warning and keep ModuleMap's existing WIN_1252
default instead of aborting macro extraction, matching how MAPIMessage
and HPSF Property handle unsupported code pages.
Closes #1515
Generated-by: Google Gemini
(cherry picked from commit f0f8c91a084f4875bd2107b9b3d0a98f731d96b1)
---------
Co-authored-by: nikhil1511 <[email protected]>
---
.../test/java/org/apache/poi/hsmf/TestBasics.java | 4 +-
.../apache/poi/poifs/macros/VBAMacroReader.java | 14 ++-
.../java/org/apache/poi/util/CodePageUtil.java | 21 +++-
.../poi/poifs/macros/TestVBAMacroReader.java | 117 +++++++++++++++++++++
4 files changed, 150 insertions(+), 6 deletions(-)
diff --git a/poi-scratchpad/src/test/java/org/apache/poi/hsmf/TestBasics.java
b/poi-scratchpad/src/test/java/org/apache/poi/hsmf/TestBasics.java
index 95185d39b7..913d0f8ab2 100644
--- a/poi-scratchpad/src/test/java/org/apache/poi/hsmf/TestBasics.java
+++ b/poi-scratchpad/src/test/java/org/apache/poi/hsmf/TestBasics.java
@@ -252,9 +252,9 @@ public final class TestBasics {
// Defaults to CP1251
assertEquals("CP1252",
chinese.getRecipientDetailsChunks()[0].getRecipientDisplayNameChunk().get7BitEncoding());
- // But after guessing goes to the correct one, cp950 (Windows
Traditional Chinese)
+ // But after guessing goes to the correct one, MS950 (Microsoft's
Windows Traditional Chinese Big5 variant)
chinese.guess7BitEncoding();
- assertEquals("cp950",
chinese.getRecipientDetailsChunks()[0].getRecipientDisplayNameChunk().get7BitEncoding());
+ assertEquals("MS950",
chinese.getRecipientDetailsChunks()[0].getRecipientDisplayNameChunk().get7BitEncoding());
}
@Disabled(value = "expected to fail")
diff --git a/poi/src/main/java/org/apache/poi/poifs/macros/VBAMacroReader.java
b/poi/src/main/java/org/apache/poi/poifs/macros/VBAMacroReader.java
index dcc3a58f26..447e197d98 100644
--- a/poi/src/main/java/org/apache/poi/poifs/macros/VBAMacroReader.java
+++ b/poi/src/main/java/org/apache/poi/poifs/macros/VBAMacroReader.java
@@ -26,6 +26,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
@@ -531,11 +532,20 @@ public class VBAMacroReader implements Closeable {
case PROJECT_CODEPAGE:
in.readInt();//record size must == 4
int codepage = in.readShort();
- modules.charset =
Charset.forName(CodePageUtil.codepageToEncoding(codepage, true));
+ try {
+ modules.charset =
Charset.forName(CodePageUtil.codepageToEncoding(codepage, true));
+ } catch (UnsupportedEncodingException |
IllegalArgumentException e) {
+ LOGGER.atWarn().log(
+ "Unsupported VBA project code page {},
decoding MBCS strings as windows-1252",
+ box(codepage));
+ modules.charset = WIN_1252;
+ }
break;
case MODULE_STREAM_NAME:
ASCIIUnicodeStringPair pair = readStringPair(in,
modules.charset, STREAMNAME_RESERVED);
- streamName = pair.ascii();
+ // [MS-OVBA] 2.3.4.2.3.2.3: StreamNameUnicode MUST
contain the UTF-16 encoding
+ // of StreamName, matching the UTF-16 compound
file entry name ([MS-CFB] 2.6.1).
+ streamName = pair.unicode().isEmpty() ?
pair.ascii() : pair.unicode();
break;
case PROJECT_DOC_STRING:
readStringPair(in, modules.charset,
DOC_STRING_RESERVED);
diff --git a/poi/src/main/java/org/apache/poi/util/CodePageUtil.java
b/poi/src/main/java/org/apache/poi/util/CodePageUtil.java
index b89fba871b..ec4dd86294 100644
--- a/poi/src/main/java/org/apache/poi/util/CodePageUtil.java
+++ b/poi/src/main/java/org/apache/poi/util/CodePageUtil.java
@@ -323,7 +323,7 @@ public class CodePageUtil
return "windows-1255";
case CP_WINDOWS_1256:
if (javaLangFormat)
- return "Cp1255";
+ return "Cp1256";
else
return "windows-1256";
case CP_WINDOWS_1257:
@@ -436,7 +436,24 @@ public class CodePageUtil
case CP_GB18030:
return "GB18030";
case CP_SJIS:
- return "SJIS";
+ // Windows code page 932 is Microsoft's Shift_JIS variant (IANA
+ // "Windows-31J"). Java's "SJIS" is JIS X 0208 Shift_JIS, which
+ // lacks the NEC and IBM extensions and maps some symbols
+ // differently.
+ if (javaLangFormat) {
+ return "MS932";
+ } else {
+ return "windows-31j";
+ }
+ case 950:
+ // Windows code page 950 is Microsoft's Big5 variant. Without
+ // this case, the default "cp950" resolves to IBM's variant
+ // (x-IBM950).
+ if (javaLangFormat) {
+ return "MS950";
+ } else {
+ return "x-windows-950";
+ }
default:
return "cp" + codepage;
}
diff --git
a/poi/src/test/java/org/apache/poi/poifs/macros/TestVBAMacroReader.java
b/poi/src/test/java/org/apache/poi/poifs/macros/TestVBAMacroReader.java
index 1f2cdd7574..79df0fadae 100644
--- a/poi/src/test/java/org/apache/poi/poifs/macros/TestVBAMacroReader.java
+++ b/poi/src/test/java/org/apache/poi/poifs/macros/TestVBAMacroReader.java
@@ -18,6 +18,7 @@
package org.apache.poi.poifs.macros;
import org.apache.poi.POIDataSamples;
+import org.apache.poi.poifs.filesystem.DirectoryNode;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import org.apache.poi.util.IOUtils;
import org.apache.poi.util.StringUtil;
@@ -463,6 +464,122 @@ class TestVBAMacroReader {
return bos.toByteArray();
}
+ @Test
+ void moduleMapKeyedByStreamNameUnicode() throws IOException {
+ final String unicodeStreamName = "\u9ad9\u6a4b\u2460";
+ final String source = "Attribute VB_Name = \"Mod1\"\r\nSub
Run()\r\nEnd Sub\r\n";
+ // Simulate an MBCS StreamName that does not round-trip to the UTF-16
entry name
+ final byte[] mbcsName = "???".getBytes(StandardCharsets.US_ASCII);
+ final byte[] utf16Name =
unicodeStreamName.getBytes(StandardCharsets.UTF_16LE);
+
+ final ByteArrayOutputStream dir = new ByteArrayOutputStream();
+ // PROJECT_CODEPAGE = 1252
+ writeShort(dir, 0x0003);
+ writeInt(dir, 2);
+ writeShort(dir, 1252);
+ // MODULE_STREAM_NAME (MBCS + 0x0032 + UTF-16LE)
+ writeShort(dir, 0x001A);
+ writeInt(dir, mbcsName.length);
+ dir.writeBytes(mbcsName);
+ writeShort(dir, 0x0032);
+ writeInt(dir, utf16Name.length);
+ dir.writeBytes(utf16Name);
+ // MODULE_OFFSET = 0
+ writeShort(dir, 0x0031);
+ writeInt(dir, 4);
+ writeInt(dir, 0);
+ // DIR_STREAM_TERMINATOR
+ writeShort(dir, 0x0010);
+ writeInt(dir, 0);
+
+ final ByteArrayOutputStream projectWm = new ByteArrayOutputStream();
+ projectWm.writeBytes(mbcsName);
+ projectWm.write(0);
+ projectWm.writeBytes(utf16Name);
+ projectWm.write(new byte[]{0, 0, 0, 0});
+
+ try (POIFSFileSystem fs = new POIFSFileSystem();
+ VBAMacroReader reader = new VBAMacroReader(fs)) {
+ DirectoryNode vba = (DirectoryNode)
fs.getRoot().createDirectory("VBA");
+ vba.createDocument("dir", new
ByteArrayInputStream(compressLiterals(dir.toByteArray())));
+ vba.createDocument(unicodeStreamName, new ByteArrayInputStream(
+
compressLiterals(source.getBytes(StandardCharsets.US_ASCII))));
+ fs.getRoot().createDocument("PROJECTwm", new
ByteArrayInputStream(projectWm.toByteArray()));
+ fs.getRoot().createDocument("PROJECT", new ByteArrayInputStream(
+ "Module=???\r\n".getBytes(StandardCharsets.US_ASCII)));
+
+ Map<String, String> macros = reader.readMacros();
+ assertEquals(Collections.singletonMap(unicodeStreamName, source),
macros);
+
+ Map<String, Module> modules = reader.readMacroModules();
+ assertEquals(Module.ModuleType.Module,
modules.get(unicodeStreamName).geModuleType());
+ }
+ }
+
+ @Test
+ void unsupportedProjectCodepageFallsBackToWindows1252() throws IOException
{
+ final String source = "Attribute VB_Name = \"Module1\"\r\nSub
Run()\r\nMsgBox \"\u00e9\"\r\nEnd Sub\r\n";
+ final byte[] sourceBytes = source.getBytes(StringUtil.WIN_1252);
+ final byte[] asciiName = "Module1".getBytes(StandardCharsets.US_ASCII);
+ final byte[] utf16Name = "Module1".getBytes(StandardCharsets.UTF_16LE);
+
+ // Test both an unknown positive code page
(UnsupportedCharsetException)
+ // and code page 0 (UnsupportedEncodingException from CodePageUtil).
+ for (int unsupportedCodepage : new int[]{10025, 0}) {
+ final ByteArrayOutputStream dir = new ByteArrayOutputStream();
+ writeShort(dir, 0x0003);
+ writeInt(dir, 2);
+ writeShort(dir, unsupportedCodepage);
+ writeShort(dir, 0x001A);
+ writeInt(dir, asciiName.length);
+ dir.writeBytes(asciiName);
+ writeShort(dir, 0x0032);
+ writeInt(dir, utf16Name.length);
+ dir.writeBytes(utf16Name);
+ writeShort(dir, 0x0031);
+ writeInt(dir, 4);
+ writeInt(dir, 0);
+ writeShort(dir, 0x0010);
+ writeInt(dir, 0);
+
+ try (POIFSFileSystem fs = new POIFSFileSystem();
+ VBAMacroReader reader = new VBAMacroReader(fs)) {
+ DirectoryNode vba = (DirectoryNode)
fs.getRoot().createDirectory("VBA");
+ vba.createDocument("dir", new
ByteArrayInputStream(compressLiterals(dir.toByteArray())));
+ vba.createDocument("Module1", new
ByteArrayInputStream(compressLiterals(sourceBytes)));
+
+ Map<String, String> macros = reader.readMacros();
+ assertEquals(Collections.singletonMap("Module1", source),
macros);
+ }
+ }
+ }
+
+ /**
+ * RLE-compresses {@code data} into a single chunk of literals only
+ */
+ private static byte[] compressLiterals(byte[] data) {
+ final ByteArrayOutputStream chunk = new ByteArrayOutputStream();
+ for (int i = 0; i < data.length; i += 8) {
+ chunk.write(0);
+ chunk.write(data, i, Math.min(8, data.length - i));
+ }
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ out.write(0x01);
+ writeShort(out, 0xB000 | (chunk.size() - 1));
+ out.writeBytes(chunk.toByteArray());
+ return out.toByteArray();
+ }
+
+ private static void writeShort(ByteArrayOutputStream out, int value) {
+ out.write(value);
+ out.write(value >> 8);
+ }
+
+ private static void writeInt(ByteArrayOutputStream out, int value) {
+ writeShort(out, value);
+ writeShort(out, value >> 16);
+ }
+
private static void putZipEntry(ZipOutputStream zos, String name, byte[]
data) throws IOException {
zos.putNextEntry(new ZipEntry(name));
zos.write(data);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]