This is an automated email from the ASF dual-hosted git repository.

markt-asf pushed a commit to branch 10.1.x
in repository https://gitbox.apache.org/repos/asf/tomcat.git


The following commit(s) were added to refs/heads/10.1.x by this push:
     new ecaf5bc81e Expand the user name normalisation options in LockOutRealm
ecaf5bc81e is described below

commit ecaf5bc81e55591971d43820f60a892e100086d0
Author: Mark Thomas <[email protected]>
AuthorDate: Fri Nov 20 17:41:24 2026 +0000

    Expand the user name normalisation options in LockOutRealm
    
    Add options to ignore invisible characters (charactersToIgnore with a
    default set and additionalCharactersToIgnore), convert user names to
    NFKC (normalizeUnicode), remove combining marks (stripCombiningMarks)
    and collapse internal space (collapseInternalSpace). All are enabled by
    default so that user names that LDAP servers and databases treat as the
    same user are treated as the same user when making lock out decisions.
    
    The character options support \uXXXX, \u{X...X} and \\ escape sequences
    and return their value in the same escaped form.
    
    Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
---
 .../apache/catalina/realm/LocalStrings.properties  |   1 +
 java/org/apache/catalina/realm/LockOutRealm.java   | 443 ++++++++++++++++++++-
 .../apache/catalina/realm/TestLockoutRealm.java    | 421 ++++++++++++++++++++
 webapps/docs/changelog.xml                         |   4 +
 webapps/docs/config/realm.xml                      | 100 ++++-
 5 files changed, 956 insertions(+), 13 deletions(-)

diff --git a/java/org/apache/catalina/realm/LocalStrings.properties 
b/java/org/apache/catalina/realm/LocalStrings.properties
index e80fb53823..5a3b42b7ba 100644
--- a/java/org/apache/catalina/realm/LocalStrings.properties
+++ b/java/org/apache/catalina/realm/LocalStrings.properties
@@ -96,6 +96,7 @@ jndiRealm.open=Exception opening directory server connection
 jndiRealm.tlsClose=Exception closing tls response
 
 lockOutRealm.authLockedUser=An attempt was made to authenticate the locked 
user [{0}]
+lockOutRealm.invalidEscape=The value [{0}] is not valid. The supported escape 
sequences are \\\\, \\uXXXX and \\u'{'X...X'}' (one to six hex digits) and 
unpaired surrogates are not permitted
 lockOutRealm.removeWarning=User [{0}] was removed from the failed users cache 
after [{1}] seconds to keep the cache size within the limit set
 
 mdCredentialHandler.unknownEncoding=The encoding [{0}] is not supported so the 
current setting of [{1}] will still be used
diff --git a/java/org/apache/catalina/realm/LockOutRealm.java 
b/java/org/apache/catalina/realm/LockOutRealm.java
index e2931d3386..8ced3cef73 100644
--- a/java/org/apache/catalina/realm/LockOutRealm.java
+++ b/java/org/apache/catalina/realm/LockOutRealm.java
@@ -18,14 +18,18 @@ package org.apache.catalina.realm;
 
 import java.security.Principal;
 import java.security.cert.X509Certificate;
+import java.text.Normalizer;
 import java.util.LinkedHashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
 
 import org.apache.catalina.LifecycleException;
 import org.apache.juli.logging.Log;
 import org.apache.juli.logging.LogFactory;
+import org.apache.tomcat.util.buf.HexUtils;
 import org.ietf.jgss.GSSContext;
 import org.ietf.jgss.GSSCredential;
 import org.ietf.jgss.GSSException;
@@ -49,6 +53,21 @@ public class LockOutRealm extends CombinedRealm {
 
     private static final Log log = LogFactory.getLog(LockOutRealm.class);
 
+    private static final String DEFAULT_CHARACTERS_TO_IGNORE;
+
+    private static final Set<Integer> DEFAULT_CODE_POINTS_TO_IGNORE;
+
+    static {
+        StringBuilder sb = new StringBuilder();
+        for (int cp = 0; cp <= Character.MAX_CODE_POINT; cp++) {
+            if (isIgnoredByDefault(cp)) {
+                sb.appendCodePoint(cp);
+            }
+        }
+        DEFAULT_CHARACTERS_TO_IGNORE = sb.toString();
+        DEFAULT_CODE_POINTS_TO_IGNORE = 
toCodePointSet(DEFAULT_CHARACTERS_TO_IGNORE);
+    }
+
     /**
      * The number of times in a row a user has to fail authentication to be 
locked out. Defaults to 5.
      */
@@ -93,6 +112,44 @@ public class LockOutRealm extends CombinedRealm {
      */
     private boolean stripTrailingSpace = true;
 
+    /*
+     * Characters that, if present in a user name, will be treated as if they 
are not present in the user name for the
+     * purposes of this realm. The default is the set of characters that LDAP 
servers and most databases ignore when
+     * comparing user names (see isIgnoredByDefault()). This is the actual 
characters, not the escaped form.
+     */
+    private volatile String charactersToIgnore = DEFAULT_CHARACTERS_TO_IGNORE;
+
+    /*
+     * Further characters that are treated as if they are not present in the 
user name. These are in addition to
+     * charactersToIgnore. The default is none. This is the actual characters, 
not the escaped form.
+     */
+    private volatile String additionalCharactersToIgnore = "";
+
+    /*
+     * The code points of the characters in charactersToIgnore and 
additionalCharactersToIgnore. Derived from them when
+     * either is set so it does not need to be derived each time a user name 
is normalized.
+     */
+    private volatile Set<Integer> codePointsToIgnore = 
DEFAULT_CODE_POINTS_TO_IGNORE;
+
+    /*
+     * Should user names (the keys) in the failedUsers Map be converted to 
Unicode Normalization Form KC (NFKC). This
+     * is the form used by LDAP servers (RFC 4518, RFC 4013) when comparing 
user names. The default is true.
+     */
+    private volatile boolean normalizeUnicode = true;
+
+    /*
+     * Should combining marks (Unicode category Mn) be removed from user names 
(the keys) in the failedUsers Map. This
+     * matches the behaviour of accent insensitive database collations. The 
default is true.
+     */
+    private volatile boolean stripCombiningMarks = true;
+
+    /*
+     * Should runs of space characters within user names (the keys) in the 
failedUsers Map be replaced with a single
+     * space. The default is true.
+     */
+    private volatile boolean collapseInternalSpace = true;
+
+
     @Override
     protected void startInternal() throws LifecycleException {
         /*
@@ -424,39 +481,411 @@ public class LockOutRealm extends CombinedRealm {
     }
 
 
+    /**
+     * Get the characters that are ignored in user names when making locking 
decisions. This does not include the
+     * additional characters to ignore.
+     * <p>
+     * The value is returned in escaped form so it can be passed to {@link 
#setCharactersToIgnore(String)}. Characters
+     * other than the printable ASCII characters (U+0021 to U+007E) are 
returned as {@code \}{@code uXXXX} (four hex
+     * digits) or, for characters outside the Basic Multilingual Plane, {@code 
\}{@code u{X...X}}. A backslash is
+     * returned as two backslashes.
+     *
+     * @return the characters that are ignored in escaped form
+     */
+    public String getCharactersToIgnore() {
+        return escape(charactersToIgnore);
+    }
+
+
+    /**
+     * Set the characters that are ignored in user names when making locking 
decisions. Each Unicode character
+     * (code point) in the provided value is removed from user names. The 
provided value replaces the default value in
+     * its entirety. A {@code null} value is treated as an empty String, 
meaning that no characters are ignored. The
+     * characters set with {@link #setAdditionalCharactersToIgnore(String)} 
are ignored as well.
+     * <p>
+     * Characters that are difficult to specify directly may be escaped. The 
supported escape sequences are:
+     * <ul>
+     * <li>{@code \}{@code \} - a backslash</li>
+     * <li>{@code \}{@code uXXXX} - the UTF-16 code unit with the four digit 
hex value XXXX (a pair of these escape
+     * sequences may be used to specify a character outside the Basic 
Multilingual Plane)</li>
+     * <li>{@code \}{@code u{X...X}} - the character with the one to six digit 
hex value X...X</li>
+     * </ul>
+     * Any other use of a backslash is an error as is an unpaired surrogate.
+     * <p>
+     * The default value is every Unicode format character (category Cf, 
includes soft hyphen and the zero width
+     * characters), every control character (category Cc) that is not a space 
character, and the following characters
+     * that LDAP servers and many databases also ignore when comparing values: 
U+034F, U+1806, U+180B to U+180D,
+     * U+FE00 to U+FE0F and U+FFFC.
+     *
+     * @param charactersToIgnore the characters to ignore, optionally 
containing escape sequences
+     *
+     * @throws IllegalArgumentException if the provided value contains an 
invalid escape sequence or an unpaired
+     *                                      surrogate
+     */
+    public void setCharactersToIgnore(String charactersToIgnore) {
+        String characters = unescape(charactersToIgnore);
+        synchronized (this) {
+            this.charactersToIgnore = characters;
+            updateCodePointsToIgnore();
+        }
+    }
+
+
+    /**
+     * Get the additional characters that are ignored in user names when 
making locking decisions.
+     * <p>
+     * The value is returned in escaped form so it can be passed to {@link 
#setAdditionalCharactersToIgnore(String)}.
+     * See {@link #getCharactersToIgnore()} for the form used.
+     *
+     * @return the additional characters that are ignored in escaped form
+     */
+    public String getAdditionalCharactersToIgnore() {
+        return escape(additionalCharactersToIgnore);
+    }
+
+
+    /**
+     * Set the additional characters that are ignored in user names when 
making locking decisions. These characters are
+     * ignored in addition to those set with {@link 
#setCharactersToIgnore(String)} (or the default value of that
+     * attribute if it has not been set). A {@code null} value is treated as 
an empty String, meaning that there are no
+     * additional characters. The same escape sequences are supported as for 
{@link #setCharactersToIgnore(String)}.
+     *
+     * @param additionalCharactersToIgnore the additional characters to 
ignore, optionally containing escape sequences
+     *
+     * @throws IllegalArgumentException if the provided value contains an 
invalid escape sequence or an unpaired
+     *                                      surrogate
+     */
+    public void setAdditionalCharactersToIgnore(String 
additionalCharactersToIgnore) {
+        String characters = unescape(additionalCharactersToIgnore);
+        synchronized (this) {
+            this.additionalCharactersToIgnore = characters;
+            updateCodePointsToIgnore();
+        }
+    }
+
+
+    private void updateCodePointsToIgnore() {
+        codePointsToIgnore = 
toCodePointSet(charactersToIgnore.concat(additionalCharactersToIgnore));
+    }
+
+
+    /**
+     * Get the Unicode normalization flag for user name matching.
+     *
+     * @return {@code true} if user names are converted to Unicode 
Normalization Form KC (NFKC)
+     */
+    public boolean getNormalizeUnicode() {
+        return normalizeUnicode;
+    }
+
+
+    /**
+     * Set the Unicode normalization flag for user name matching. If {@code 
true}, user names are converted to Unicode
+     * Normalization Form KC (NFKC) when making locking decisions. This is the 
form used by LDAP servers.
+     *
+     * @param normalizeUnicode the normalizeUnicode to set
+     */
+    public void setNormalizeUnicode(boolean normalizeUnicode) {
+        this.normalizeUnicode = normalizeUnicode;
+    }
+
+
+    /**
+     * Get the strip combining marks flag for user name matching.
+     *
+     * @return {@code true} if combining marks are removed from user names
+     */
+    public boolean getStripCombiningMarks() {
+        return stripCombiningMarks;
+    }
+
+
+    /**
+     * Set the strip combining marks flag for user name matching. If {@code 
true}, nonspacing combining marks (Unicode
+     * category Mn, such as accents) are removed from user names when making 
locking decisions. Accent insensitive
+     * database collations treat user names that differ only by such marks as 
the same user name.
+     *
+     * @param stripCombiningMarks the stripCombiningMarks to set
+     */
+    public void setStripCombiningMarks(boolean stripCombiningMarks) {
+        this.stripCombiningMarks = stripCombiningMarks;
+    }
+
+
+    /**
+     * Get the collapse internal space flag for user name matching.
+     *
+     * @return {@code true} if runs of space within user names are replaced 
with a single space
+     */
+    public boolean getCollapseInternalSpace() {
+        return collapseInternalSpace;
+    }
+
+
+    /**
+     * Set the collapse internal space flag for user name matching. If {@code 
true}, each run of one or more space
+     * characters (see {@link #setStripLeadingSpace(boolean)}) between other 
characters in a user name is replaced by a
+     * single space when making locking decisions. Leading and trailing space 
is controlled by the strip leading space
+     * and strip trailing space flags.
+     *
+     * @param collapseInternalSpace the collapseInternalSpace to set
+     */
+    public void setCollapseInternalSpace(boolean collapseInternalSpace) {
+        this.collapseInternalSpace = collapseInternalSpace;
+    }
+
+
     private String normalizeUsername(String username) {
         if (username == null) {
             return null;
         }
+
+        // Remove ignored characters first so they can't prevent characters 
being combined or collapsed below
+        Set<Integer> remove = codePointsToIgnore;
+        if (!remove.isEmpty()) {
+            username = username.codePoints().filter(cp -> 
!remove.contains(Integer.valueOf(cp)))
+                    .collect(StringBuilder::new, 
StringBuilder::appendCodePoint, StringBuilder::append).toString();
+        }
+
+        // Normalize before changing case since normalization can change the 
case of a character
+        if (getNormalizeUnicode()) {
+            username = Normalizer.normalize(username, Normalizer.Form.NFKC);
+        }
+
         if (!getCaseSensitive()) {
             username = username.toLowerCase(Locale.ROOT);
         }
 
-        // String.strip() only covers Character.isWhitespace(). 
Character.isSpaceChar() is also needed here.
+        if (getStripCombiningMarks()) {
+            username = removeCombiningMarks(username);
+        }
+
+        if (getCollapseInternalSpace()) {
+            username = collapseInternalSpace(username);
+        }
+
         int start = 0;
         int end = username.length();
         if (getStripLeadingSpace()) {
             while (start < end) {
-                char c = username.charAt(start);
-                if (!Character.isSpaceChar(c) && !Character.isWhitespace(c)) {
+                int cp = username.codePointAt(start);
+                if (!isSpace(cp)) {
                     break;
                 }
-                start++;
+                start += Character.charCount(cp);
             }
         }
         if (getStripTrailingSpace()) {
             while (end > start) {
-                char c = username.charAt(end - 1);
-                if (!Character.isSpaceChar(c) && !Character.isWhitespace(c)) {
+                int cp = username.codePointBefore(end);
+                if (!isSpace(cp)) {
                     break;
                 }
-                end--;
+                end -= Character.charCount(cp);
             }
         }
         return username.substring(start, end);
     }
 
 
+    private String removeCombiningMarks(String username) {
+        String decomposed = Normalizer.normalize(username, 
Normalizer.Form.NFD);
+        StringBuilder sb = new StringBuilder(decomposed.length());
+        decomposed.codePoints().filter(cp -> Character.getType(cp) != 
Character.NON_SPACING_MARK)
+                .forEach(sb::appendCodePoint);
+        // Recompose what is left
+        return Normalizer.normalize(sb, getNormalizeUnicode() ? 
Normalizer.Form.NFKC : Normalizer.Form.NFC);
+    }
+
+
+    private String collapseInternalSpace(String username) {
+        int length = username.length();
+
+        // Leading and trailing space is not internal space. It is left for 
the strip options to deal with.
+        int start = 0;
+        while (start < length) {
+            int cp = username.codePointAt(start);
+            if (!isSpace(cp)) {
+                break;
+            }
+            start += Character.charCount(cp);
+        }
+        if (start == length) {
+            return username;
+        }
+        int end = length;
+        while (end > start) {
+            int cp = username.codePointBefore(end);
+            if (!isSpace(cp)) {
+                break;
+            }
+            end -= Character.charCount(cp);
+        }
+
+        StringBuilder sb = new StringBuilder(length);
+        sb.append(username, 0, start);
+        boolean inSpace = false;
+        int i = start;
+        while (i < end) {
+            int cp = username.codePointAt(i);
+            if (isSpace(cp)) {
+                if (!inSpace) {
+                    sb.append(' ');
+                    inSpace = true;
+                }
+            } else {
+                sb.appendCodePoint(cp);
+                inSpace = false;
+            }
+            i += Character.charCount(cp);
+        }
+        sb.append(username, end, length);
+        return sb.toString();
+    }
+
+
+    /*
+     * Character.isWhitespace() only covers some space characters. 
Character.isSpaceChar() is also needed. U+0085 (NEL)
+     * is neither but LDAP servers treat it as a space.
+     */
+    private static boolean isSpace(int cp) {
+        return Character.isWhitespace(cp) || Character.isSpaceChar(cp) || cp 
== 0x0085;
+    }
+
+
+    /*
+     * The characters that are ignored by default. These are characters that 
LDAP servers ignore when comparing values
+     * (see the "map to nothing" characters in RFC 3454, table B.1, and the 
control and format characters in RFC 4518)
+     * and that many databases ignore. They are all effectively invisible.
+     */
+    private static boolean isIgnoredByDefault(int cp) {
+        switch (Character.getType(cp)) {
+            case Character.FORMAT:
+                return true;
+            case Character.CONTROL:
+                // Control characters that are space (tab, new line etc.) are 
dealt with as space
+                return !isSpace(cp);
+            default:
+                break;
+        }
+        return cp == 0x034F || cp == 0x1806 || cp >= 0x180B && cp <= 0x180D || 
cp >= 0xFE00 && cp <= 0xFE0F ||
+                cp == 0xFFFC;
+    }
+
+
+    /*
+     * Convert the escaped form of a set of characters to the characters.
+     */
+    private String unescape(String escaped) {
+        if (escaped == null) {
+            return "";
+        }
+
+        int length = escaped.length();
+        StringBuilder sb = new StringBuilder(length);
+        int i = 0;
+        while (i < length) {
+            char c = escaped.charAt(i++);
+            if (c != '\\') {
+                sb.append(c);
+                continue;
+            }
+            if (i == length) {
+                throw invalidEscape(escaped);
+            }
+            char type = escaped.charAt(i++);
+            if (type == '\\') {
+                sb.append('\\');
+            } else if (type == 'u') {
+                if (i < length && escaped.charAt(i) == '{') {
+                    int close = escaped.indexOf('}', i);
+                    if (close < 0) {
+                        throw invalidEscape(escaped);
+                    }
+                    int cp = parseHex(escaped, i + 1, close, 6);
+                    if (cp > Character.MAX_CODE_POINT) {
+                        throw invalidEscape(escaped);
+                    }
+                    sb.appendCodePoint(cp);
+                    i = close + 1;
+                } else {
+                    if (i + 4 > length) {
+                        throw invalidEscape(escaped);
+                    }
+                    sb.append((char) parseHex(escaped, i, i + 4, 4));
+                    i += 4;
+                }
+            } else {
+                throw invalidEscape(escaped);
+            }
+        }
+
+        // Surrogates are only valid if paired. Pairs are always read as a 
single code point.
+        String result = sb.toString();
+        int j = 0;
+        while (j < result.length()) {
+            int cp = result.codePointAt(j);
+            if (Character.isSurrogate((char) cp) && cp <= 0xFFFF) {
+                throw invalidEscape(escaped);
+            }
+            j += Character.charCount(cp);
+        }
+        return result;
+    }
+
+
+    /*
+     * Parse the hex digits in value from start (inclusive) to end 
(exclusive). At least one digit is required and no
+     * more than maxDigits are permitted. Only the ASCII hex digits are 
permitted.
+     */
+    private int parseHex(String value, int start, int end, int maxDigits) {
+        if (end <= start || end - start > maxDigits) {
+            throw invalidEscape(value);
+        }
+        int result = 0;
+        for (int i = start; i < end; i++) {
+            int digit = HexUtils.getDec(value.charAt(i));
+            if (digit < 0) {
+                throw invalidEscape(value);
+            }
+            result = result * 16 + digit;
+        }
+        return result;
+    }
+
+
+    private IllegalArgumentException invalidEscape(String value) {
+        return new 
IllegalArgumentException(sm.getString("lockOutRealm.invalidEscape", value));
+    }
+
+
+    /*
+     * Convert a set of characters to the escaped form. Only the printable 
ASCII characters other than the backslash
+     * are not escaped. This avoids any ambiguity when displaying or editing 
the value.
+     */
+    private static String escape(String characters) {
+        StringBuilder sb = new StringBuilder(characters.length());
+        characters.codePoints().forEach(cp -> {
+            if (cp == '\\') {
+                sb.append("\\\\");
+            } else if (cp >= 0x21 && cp <= 0x7E) {
+                sb.appendCodePoint(cp);
+            } else if (cp <= 0xFFFF) {
+                String hex = Integer.toHexString(cp).toUpperCase(Locale.ROOT);
+                sb.append("\\u").append("0000", hex.length(), 4).append(hex);
+            } else {
+                
sb.append("\\u{").append(Integer.toHexString(cp).toUpperCase(Locale.ROOT)).append('}');
+            }
+        });
+        return sb.toString();
+    }
+
+
+    private static Set<Integer> toCodePointSet(String characters) {
+        return 
characters.codePoints().boxed().collect(Collectors.toUnmodifiableSet());
+    }
+
+
     /**
      * Internal record to track lock state for a user.
      */
diff --git a/test/org/apache/catalina/realm/TestLockoutRealm.java 
b/test/org/apache/catalina/realm/TestLockoutRealm.java
index d859fc85b4..d3e1590e67 100644
--- a/test/org/apache/catalina/realm/TestLockoutRealm.java
+++ b/test/org/apache/catalina/realm/TestLockoutRealm.java
@@ -26,6 +26,7 @@ import org.junit.Test;
 import org.apache.catalina.Context;
 import org.apache.catalina.startup.TesterMapRealm;
 import org.apache.tomcat.unittest.TesterContext;
+import org.apache.tomcat.util.IntrospectionUtils;
 
 public class TestLockoutRealm {
 
@@ -114,4 +115,424 @@ public class TestLockoutRealm {
         p = realm.authenticate(USER_NAME, PASSWORD);
         Assert.assertNull(p);
     }
+
+
+    @Test
+    public void testDefaults() {
+        Assert.assertTrue(realm.getNormalizeUnicode());
+        Assert.assertTrue(realm.getStripCombiningMarks());
+        Assert.assertTrue(realm.getCollapseInternalSpace());
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreDefault() {
+        // The value is returned in escaped form
+        String defaults = realm.getCharactersToIgnore();
+
+        // Characters that are invisible and ignored by servers
+        Assert.assertTrue(defaults.contains("\\u00AD"));
+        Assert.assertTrue(defaults.contains("\\u200B"));
+        Assert.assertTrue(defaults.contains("\\uFEFF"));
+        Assert.assertTrue(defaults.contains("\\uFE0F"));
+        Assert.assertTrue(defaults.contains("\\u0001"));
+        Assert.assertTrue(defaults.contains("\\u{E0001}"));
+        // Characters that are significant or are space
+        Assert.assertFalse(defaults.contains("\\u002D"));
+        Assert.assertFalse(defaults.contains("\\u0020"));
+        Assert.assertFalse(defaults.contains("\\u0009"));
+        Assert.assertFalse(defaults.contains("\\u0085"));
+        Assert.assertFalse(defaults.contains("\\u00A0"));
+        Assert.assertEquals(-1, defaults.indexOf('-'));
+        // Nothing is added by default
+        Assert.assertEquals("", realm.getAdditionalCharactersToIgnore());
+
+        // A zero width space does not make this a different user name by 
default
+        realm.authenticate(USER_NAME, "wrong");
+        realm.authenticate(USER_NAME + "\u200B", "wrong");
+        // Should be locked now
+        Principal p = realm.authenticate(USER_NAME, PASSWORD);
+        Assert.assertNull(p);
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreDefaultRoundTrip() {
+        String defaults = realm.getCharactersToIgnore();
+        realm.setCharactersToIgnore(defaults);
+        Assert.assertEquals(defaults, realm.getCharactersToIgnore());
+
+        // Still ignored
+        realm.authenticate(USER_NAME, "wrong");
+        realm.authenticate(USER_NAME + "\u200B", "wrong");
+        Assert.assertTrue(realm.isLocked(USER_NAME));
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreEmpty() {
+        realm.setCharactersToIgnore("");
+        Assert.assertEquals("", realm.getCharactersToIgnore());
+
+        // A zero width space makes this a different user name
+        realm.authenticate(USER_NAME, "wrong");
+        realm.authenticate(USER_NAME + "\u200B", "wrong");
+        // Should not be locked yet
+        Principal p = realm.authenticate(USER_NAME, PASSWORD);
+        Assert.assertNotNull(p);
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreConfigured() {
+        realm.setCharactersToIgnore("\u200B");
+        Assert.assertEquals("\\u200B", realm.getCharactersToIgnore());
+
+        realm.authenticate(USER_NAME, "wrong");
+        realm.authenticate(USER_NAME + "\u200B", "wrong");
+        // Should be locked now
+        Principal p = realm.authenticate(USER_NAME, PASSWORD);
+        Assert.assertNull(p);
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreReplacesDefault() {
+        realm.setCharactersToIgnore("\u200B");
+
+        // The soft hyphen is no longer ignored
+        realm.authenticate(USER_NAME, "wrong");
+        realm.authenticate(USER_NAME + "\u00AD", "wrong");
+        // Should not be locked yet
+        Principal p = realm.authenticate(USER_NAME, PASSWORD);
+        Assert.assertNotNull(p);
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreAnywhereInUserName() {
+        realm.setCharactersToIgnore("\u200B\u00AD");
+
+        realm.authenticate("us\u00ADer", "wrong");
+        realm.authenticate("\u200Buser\u200B", "wrong");
+        // Should be locked now
+        Principal p = realm.authenticate(USER_NAME, PASSWORD);
+        Assert.assertNull(p);
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreSupplementaryCharacters() {
+        // U+1F600 is stored as a surrogate pair. U+1F60E shares the first 
half of that pair.
+        realm.setCharactersToIgnore("\uD83D\uDE00");
+
+        realm.authenticate(USER_NAME, "wrong");
+        realm.authenticate(USER_NAME + "\uD83D\uDE0E", "wrong");
+        // The second user name is different so should not be locked yet
+        Principal p = realm.authenticate(USER_NAME, PASSWORD);
+        Assert.assertNotNull(p);
+
+        realm.authenticate(USER_NAME, "wrong");
+        realm.authenticate(USER_NAME + "\uD83D\uDE00", "wrong");
+        // The second user name is the same so should be locked now
+        p = realm.authenticate(USER_NAME, PASSWORD);
+        Assert.assertNull(p);
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreNull() {
+        realm.setCharactersToIgnore("\u200B");
+        realm.setCharactersToIgnore(null);
+        Assert.assertEquals("", realm.getCharactersToIgnore());
+
+        realm.authenticate(USER_NAME, "wrong");
+        realm.authenticate(USER_NAME + "\u200B", "wrong");
+        // Nothing is ignored so should not be locked
+        Principal p = realm.authenticate(USER_NAME, PASSWORD);
+        Assert.assertNotNull(p);
+    }
+
+
+    @Test
+    public void testNormalizeUnicodeEnabled() {
+        // Full width forms of the user name
+        String fullWidth = "\uFF55\uFF53\uFF45\uFF52";
+
+        realm.authenticate(fullWidth, "wrong");
+        realm.authenticate(USER_NAME, "wrong");
+        Assert.assertTrue(realm.isLocked(USER_NAME));
+        Assert.assertTrue(realm.isLocked(fullWidth));
+    }
+
+
+    @Test
+    public void testNormalizeUnicodeDisabled() {
+        realm.setNormalizeUnicode(false);
+        String fullWidth = "\uFF55\uFF53\uFF45\uFF52";
+
+        realm.authenticate(fullWidth, "wrong");
+        realm.authenticate(USER_NAME, "wrong");
+        Assert.assertFalse(realm.isLocked(USER_NAME));
+        Assert.assertFalse(realm.isLocked(fullWidth));
+    }
+
+
+    @Test
+    public void testNormalizeUnicodeBeforeCase() {
+        // U+210C (black-letter capital H) has no lower case form but 
normalizes to 'H'
+        realm.authenticate("\u210Cello", "wrong");
+        realm.authenticate("hello", "wrong");
+        Assert.assertTrue(realm.isLocked("hello"));
+        Assert.assertTrue(realm.isLocked("HELLO"));
+    }
+
+
+    @Test
+    public void testStripCombiningMarksEnabled() {
+        // Precomposed and decomposed forms of u with an acute accent
+        realm.authenticate("\u00FAser", "wrong");
+        realm.authenticate("u\u0301ser", "wrong");
+        Assert.assertTrue(realm.isLocked(USER_NAME));
+    }
+
+
+    @Test
+    public void testStripCombiningMarksDisabled() {
+        realm.setStripCombiningMarks(false);
+
+        realm.authenticate("\u00FAser", "wrong");
+        realm.authenticate("u\u0301ser", "wrong");
+        // Normalization still treats the two forms as the same but they are 
not the same as the plain user name
+        Assert.assertTrue(realm.isLocked("\u00FAser"));
+        Assert.assertFalse(realm.isLocked(USER_NAME));
+    }
+
+
+    @Test
+    public void testCollapseInternalSpaceEnabled() {
+        realm.authenticate("my  user", "wrong");
+        realm.authenticate("my\u00A0\tuser", "wrong");
+        Assert.assertTrue(realm.isLocked("my user"));
+    }
+
+
+    @Test
+    public void testCollapseInternalSpaceDisabled() {
+        realm.setCollapseInternalSpace(false);
+
+        realm.authenticate("my  user", "wrong");
+        realm.authenticate("my\u00A0\tuser", "wrong");
+        Assert.assertFalse(realm.isLocked("my user"));
+        Assert.assertFalse(realm.isLocked("my  user"));
+    }
+
+
+    @Test
+    public void 
testCollapseInternalSpaceLeavesLeadingAndTrailingSpaceToStripOptions() {
+        realm.setStripLeadingSpace(false);
+        realm.setStripTrailingSpace(false);
+
+        realm.authenticate("  my  user  ", "wrong");
+        realm.authenticate("  my user  ", "wrong");
+        // Internal space was collapsed but leading and trailing space was 
left alone
+        Assert.assertTrue(realm.isLocked("  my user  "));
+        Assert.assertFalse(realm.isLocked(" my user "));
+        Assert.assertFalse(realm.isLocked("my user"));
+    }
+
+
+    @Test
+    public void testIgnoredCharactersAreRemovedBeforeSpaceIsCollapsed() {
+        realm.authenticate("my \u200B user", "wrong");
+        realm.authenticate("my user", "wrong");
+        Assert.assertTrue(realm.isLocked("my user"));
+    }
+
+
+    @Test
+    public void testNextLineIsSpace() {
+        realm.authenticate("my\u0085user", "wrong");
+        realm.authenticate("my user", "wrong");
+        Assert.assertTrue(realm.isLocked("my user"));
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreEscapeUnicode() {
+        realm.setCharactersToIgnore("\\u200B\\u00ad");
+        Assert.assertEquals("\\u200B\\u00AD", realm.getCharactersToIgnore());
+
+        realm.authenticate("us\u00ADer", "wrong");
+        realm.authenticate("user\u200B", "wrong");
+        Assert.assertTrue(realm.isLocked(USER_NAME));
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreEscapeBraces() {
+        realm.setCharactersToIgnore("\\u{200B}\\u{1F600}");
+        Assert.assertEquals("\\u200B\\u{1F600}", 
realm.getCharactersToIgnore());
+
+        realm.authenticate("us\uD83D\uDE00er", "wrong");
+        realm.authenticate("user\u200B", "wrong");
+        Assert.assertTrue(realm.isLocked(USER_NAME));
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreEscapeSurrogatePair() {
+        realm.setCharactersToIgnore("\\uD83D\\uDE00");
+        // The pair is a single character
+        Assert.assertEquals("\\u{1F600}", realm.getCharactersToIgnore());
+
+        realm.authenticate(USER_NAME, "wrong");
+        realm.authenticate(USER_NAME + "\uD83D\uDE0E", "wrong");
+        // Different user name
+        Assert.assertFalse(realm.isLocked(USER_NAME));
+        realm.authenticate(USER_NAME + "\uD83D\uDE00", "wrong");
+        Assert.assertTrue(realm.isLocked(USER_NAME));
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreEscapeBackslash() {
+        realm.setCharactersToIgnore("\\\\");
+        Assert.assertEquals("\\\\", realm.getCharactersToIgnore());
+
+        // A backslash is ignored but nothing else is
+        realm.authenticate("us\\er", "wrong");
+        realm.authenticate("user\u200B", "wrong");
+        Assert.assertFalse(realm.isLocked(USER_NAME));
+        realm.authenticate("\\user", "wrong");
+        Assert.assertTrue(realm.isLocked(USER_NAME));
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreEscapeControlCharacter() {
+        // Can't be written in an XML attribute value but can be escaped
+        realm.setCharactersToIgnore("\\u0001");
+
+        realm.authenticate("us\u0001er", "wrong");
+        realm.authenticate("user\u0001", "wrong");
+        Assert.assertTrue(realm.isLocked(USER_NAME));
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreGetterEscapes() {
+        // Printable ASCII is not escaped. Everything else is.
+        realm.setCharactersToIgnore("a-~ \u00E9\u200B\\\\\uD83D\uDE00\u0001");
+        Assert.assertEquals("a-~\\u0020\\u00E9\\u200B\\\\\\u{1F600}\\u0001", 
realm.getCharactersToIgnore());
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreRoundTrip() {
+        String value = "a-~ \u00E9\u200B\\\\\uD83D\uDE00\u0001";
+        realm.setCharactersToIgnore(value);
+        String escaped = realm.getCharactersToIgnore();
+
+        realm.setCharactersToIgnore("");
+        realm.setCharactersToIgnore(escaped);
+        Assert.assertEquals(escaped, realm.getCharactersToIgnore());
+
+        // And the characters are the same
+        realm.authenticate("us\u0001er", "wrong");
+        realm.authenticate("user\u00E9", "wrong");
+        Assert.assertTrue(realm.isLocked(USER_NAME));
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreInvalidEscapes() {
+        String[] invalid = { "\\", "a\\", "\\q", "\\u", "\\u20", "\\u20G0", 
"\\u{}", "\\u{200B", "\\u{110000}",
+                "\\u{1234567}", "\\u{D83D}", "\\uD83D", "\\uDE00", "\\uD83Dx", 
"\\u{ 200B}", "\\u{+200B}",
+                "\uD83D" };
+        for (String value : invalid) {
+            try {
+                realm.setCharactersToIgnore(value);
+                Assert.fail("Expected an IllegalArgumentException for [" + 
value + "]");
+            } catch (IllegalArgumentException e) {
+                // Expected
+            }
+            try {
+                realm.setAdditionalCharactersToIgnore(value);
+                Assert.fail("Expected an IllegalArgumentException for 
additional [" + value + "]");
+            } catch (IllegalArgumentException e) {
+                // Expected
+            }
+        }
+        // The values were not changed by the failed calls
+        Assert.assertTrue(realm.getCharactersToIgnore().contains("\\u200B"));
+        Assert.assertEquals("", realm.getAdditionalCharactersToIgnore());
+    }
+
+
+    @Test
+    public void testAdditionalCharactersToIgnore() {
+        realm.setAdditionalCharactersToIgnore("-");
+        Assert.assertEquals("-", realm.getAdditionalCharactersToIgnore());
+
+        // The default characters are still ignored
+        realm.authenticate("us-er", "wrong");
+        realm.authenticate("user\u200B", "wrong");
+        Assert.assertTrue(realm.isLocked(USER_NAME));
+    }
+
+
+    @Test
+    public void testAdditionalCharactersToIgnoreWithoutDefault() {
+        realm.setCharactersToIgnore("");
+        realm.setAdditionalCharactersToIgnore("\\u002D");
+        // Printable ASCII is not escaped when returned
+        Assert.assertEquals("-", realm.getAdditionalCharactersToIgnore());
+
+        realm.authenticate("us-er", "wrong");
+        realm.authenticate("user\u200B", "wrong");
+        // Only the additional character is ignored
+        Assert.assertFalse(realm.isLocked(USER_NAME));
+        realm.authenticate("user-", "wrong");
+        Assert.assertTrue(realm.isLocked(USER_NAME));
+    }
+
+
+    @Test
+    public void testAdditionalCharactersToIgnoreEscapes() {
+        realm.setCharactersToIgnore("");
+        realm.setAdditionalCharactersToIgnore("\\u{1F600}\\u0001");
+        Assert.assertEquals("\\u{1F600}\\u0001", 
realm.getAdditionalCharactersToIgnore());
+
+        realm.authenticate("us\uD83D\uDE00er", "wrong");
+        realm.authenticate("user\u0001", "wrong");
+        Assert.assertTrue(realm.isLocked(USER_NAME));
+    }
+
+
+    @Test
+    public void testAdditionalCharactersToIgnoreNull() {
+        realm.setAdditionalCharactersToIgnore("-");
+        realm.setAdditionalCharactersToIgnore(null);
+        Assert.assertEquals("", realm.getAdditionalCharactersToIgnore());
+
+        realm.authenticate("us-er", "wrong");
+        realm.authenticate(USER_NAME, "wrong");
+        Assert.assertFalse(realm.isLocked(USER_NAME));
+    }
+
+
+    @Test
+    public void testCharactersToIgnoreSetViaIntrospection() {
+        // This is how the values are set from server.xml. The escape 
sequences are in the attribute value as written
+        // (an XML parser will have already replaced any character references).
+        Assert.assertTrue(IntrospectionUtils.setProperty(realm, 
"charactersToIgnore", "\\u200B\u00AD"));
+        Assert.assertTrue(IntrospectionUtils.setProperty(realm, 
"additionalCharactersToIgnore", "\\u0001-"));
+        Assert.assertEquals("\\u200B\\u00AD", realm.getCharactersToIgnore());
+        Assert.assertEquals("\\u0001-", 
realm.getAdditionalCharactersToIgnore());
+
+        realm.authenticate("us\u0001-er", "wrong");
+        realm.authenticate("\u200Buser\u00AD", "wrong");
+        Assert.assertTrue(realm.isLocked(USER_NAME));
+    }
 }
diff --git a/webapps/docs/changelog.xml b/webapps/docs/changelog.xml
index dbe18d2fb8..37f4dac242 100644
--- a/webapps/docs/changelog.xml
+++ b/webapps/docs/changelog.xml
@@ -126,6 +126,10 @@
         <code>ClassValue</code> to cache the result of calling
         <code>Class.getMethods()</code>. (markt)
       </scode>
+      <fix>
+        Expand the user name normalisation options available in the
+        <code>LockOutRealm</code>. (markt)
+      </fix>
     </changelog>
   </subsection>
   <subsection name="Coyote">
diff --git a/webapps/docs/config/realm.xml b/webapps/docs/config/realm.xml
index 25ee65e019..f985619bdb 100644
--- a/webapps/docs/config/realm.xml
+++ b/webapps/docs/config/realm.xml
@@ -1000,6 +1000,17 @@
         one of those roles.</p>
       </attribute>
 
+      <attribute name="additionalCharactersToIgnore" required="false">
+        <p>Further characters that will be removed from user names when making
+        locking decisions. These are removed in addition to the characters
+        specified by <strong>charactersToIgnore</strong>, so the default value
+        of that attribute can be extended without having to list every
+        character it contains. The same format and escape sequences are
+        supported as for <strong>charactersToIgnore</strong>. If not specified,
+        the default value of an empty string will be used, meaning that no
+        additional characters are ignored.</p>
+      </attribute>
+
       <attribute name="cacheRemovalWarningTime" required="false">
        <p>If a failed user is removed from the cache because the cache is too
        big before it has been in the cache for at least this period of time (in
@@ -1018,6 +1029,55 @@
        treated in a case insensitive manner.</p>
       </attribute>
 
+      <attribute name="charactersToIgnore" required="false">
+        <p>The characters that will be removed from user names when making
+        locking decisions. Each Unicode character in the value is treated
+        individually; ranges are not supported. The characters are removed
+        before any other changes are made to the user name. This is used to
+        stop a user name that contains characters that the authentication
+        back-end ignores (such as zero width characters) being treated as a
+        different user to the same name without those characters. Specifying a
+        value replaces the default value in its entirety. Specify an empty
+        value to ignore no characters. To ignore characters in addition to the
+        default value, use <strong>additionalCharactersToIgnore</strong>.</p>
+        <p>Characters that are difficult to specify directly (including the
+        control characters, which can not be used in an XML attribute value
+        even as a numeric character reference) may be written as escape
+        sequences:</p>
+        <ul>
+          <li><code>\\</code> - a backslash.</li>
+          <li><code>\uXXXX</code> - the UTF-16 code unit with the four digit 
hex
+              value <code>XXXX</code>. A pair of these may be used to specify
+              a character outside the Basic Multilingual Plane.</li>
+          <li><code>\u{X...X}</code> - the character with the one to six digit
+              hex value <code>X...X</code>.</li>
+        </ul>
+        <p>Any other use of a backslash, or an unpaired surrogate, is an error.
+        When the value is read, it is always returned in this escaped form,
+        where every character other than the printable ASCII characters
+        (U+0021 to U+007E) is escaped, so the value read can be used to set
+        the value.</p>
+        <p>The default value is every Unicode format character (category
+        <code>Cf</code>, which includes the soft hyphen and the zero width
+        characters), every control character (category <code>Cc</code>) that is
+        not a space character and the characters U+034F, U+1806, U+180B to
+        U+180D, U+FE00 to U+FE0F and U+FFFC. These are the characters that LDAP
+        servers ignore when comparing values and that many databases also
+        ignore.</p>
+      </attribute>
+
+      <attribute name="collapseInternalSpace" required="false">
+        <p>Will each run of one or more space characters
+        (<code>Character.isWhitespace()</code> or
+        <code>Character.isSpaceChar()</code> returns <code>true</code>, or the
+        character is U+0085) between other characters in a user name be
+        replaced by a single space when making locking decisions. Leading and
+        trailing space is not affected by this setting; see
+        <strong>stripLeadingSpace</strong> and
+        <strong>stripTrailingSpace</strong>. Defaults to <code>true</code>,
+        meaning that runs of space will be collapsed.</p>
+      </attribute>
+
       <attribute name="failureCount" required="false">
        <p>The number of times in a row a user has to fail authentication to be
        locked out. Defaults to 5.</p>
@@ -1032,18 +1092,38 @@
        will also not reset the lock out time.</p>
       </attribute>
 
+      <attribute name="normalizeUnicode" required="false">
+        <p>Will user names be converted to Unicode Normalization Form KC (NFKC)
+        when making locking decisions. This is the form used by LDAP servers
+        when comparing user names. It means that, for example, full width and
+        half width forms of a character and the precomposed and decomposed
+        forms of an accented character are treated as the same character.
+        Defaults to <code>true</code>, meaning that user names will be
+        normalized.</p>
+      </attribute>
+
+      <attribute name="stripCombiningMarks" required="false">
+        <p>Will nonspacing combining marks (Unicode category <code>Mn</code>,
+        such as accents) be removed from user names when making locking
+        decisions. Accent insensitive database collations treat user names that
+        differ only by such marks as the same user name. Defaults to
+        <code>true</code>, meaning that combining marks will be removed.</p>
+      </attribute>
+
       <attribute name="stripLeadingSpace" required="false">
         <p>Will leading space (<code>Character.isWhitespace()</code> or
-        <code>Character.isSpaceChar()</code> returns <code>true</code>) be
-        removed from user names when making locking decisions. Defaults to
-        <code>true</code>, meaning leading space will be removed.</p>
+        <code>Character.isSpaceChar()</code> returns <code>true</code>, or the
+        character is U+0085) be removed from user names when making locking
+        decisions. Defaults to <code>true</code>, meaning leading space will be
+        removed.</p>
       </attribute>
 
       <attribute name="stripTrailingSpace" required="false">
         <p>Will trailing space (<code>Character.isWhitespace()</code> or
-        <code>Character.isSpaceChar()</code> returns <code>true</code>) be
-        removed from user names when making locking decisions. Defaults to
-        <code>true</code>, meaning leading space will be removed.</p>
+        <code>Character.isSpaceChar()</code> returns <code>true</code>, or the
+        character is U+0085) be removed from user names when making locking
+        decisions. Defaults to <code>true</code>, meaning trailing space will
+        be removed.</p>
       </attribute>
 
       <attribute name="transportGuaranteeRedirectStatus" required="false">
@@ -1055,6 +1135,14 @@
 
     </attributes>
 
+    <p>When making locking decisions, the user name is processed in the
+    following order: characters to ignore are removed, the user name is
+    converted to Unicode Normalization Form KC (if enabled), the user name is
+    converted to lower case (unless case sensitive), combining marks are 
removed
+    (if enabled), runs of internal space are collapsed (if enabled) and leading
+    and trailing space is removed (if enabled). The user name that is passed to
+    the nested Realms is not changed.</p>
+
     <p>See the <a href="../realm-howto.html">Container-Managed Security
     Guide</a> for more information on setting up container managed security
     using the LockOutRealm component.</p>


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to