[
https://issues.apache.org/jira/browse/OPENNLP-1988?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18123703#comment-18123703
]
ASF GitHub Bot commented on OPENNLP-1988:
-----------------------------------------
krickert commented on code in PR #1319:
URL: https://github.com/apache/opennlp/pull/1319#discussion_r4193623898
##########
opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java:
##########
@@ -543,30 +614,202 @@ protected static List<ChunkRange> chunkRanges(final int
tokenCount, final int do
}
/**
- * Reads a JSON vocabulary as one object mapping tokens to non-negative
integer IDs, as in
- * {@code vocab.json}. Keys are decoded from their escapes, and a later
entry for the same
- * token overwrites an earlier one.
+ * Reads a JSON vocabulary in one of two layouts: a {@code vocab.json}
object that maps each
+ * token to its ID, or a {@code tokenizer.json} of a WordPiece model,
recognized by a
+ * {@code model} object with a string {@code type}, whose {@code
model.vocab} object supplies
+ * the tokens and whose {@code added_tokens} entries absent from {@code
model.vocab} are added
+ * with their ids. Keys are decoded from their escapes, and a later entry
for the same token
+ * overwrites an earlier one.
*
* @param json The JSON text of the vocabulary.
* @return A map of vocabulary tokens to IDs.
* @throws IllegalArgumentException Thrown if the text is not a single
well-formed JSON object,
- * or if a value of the vocabulary object is not a
- * non-negative integer that fits into an {@code int}. The message names
the offset or the
- * token.
+ * if a value of the vocabulary object is not a non-negative integer
that fits into an
+ * {@code int}, if a {@code tokenizer.json} is not that of a WordPiece
model with the
+ * {@code ##} continuing subword prefix and a {@code model.vocab}
object, or if an added
+ * token is malformed or conflicts with another token or id. The message
names the offset,
+ * the token, or the unsupported member.
*/
static Map<String, Integer> loadJsonVocab(final String json) {
+ return readJsonVocab(json).ids();
+ }
+
+ /**
+ * Reads a JSON vocabulary as {@link #loadJsonVocab(String)} does, keeping
the
+ * {@code normalizer.lowercase} setting of a {@code tokenizer.json}.
+ *
+ * @param json The JSON text of the vocabulary.
+ * @return The vocabulary.
+ * @throws IllegalArgumentException Thrown as by {@link
#loadJsonVocab(String)}.
+ */
+ static Vocabulary readJsonVocab(final String json) {
final List<JsonScan.Member> document = JsonScan.document(json);
+ final JsonScan.Member model = JsonScan.member(document,
TOKENIZER_MODEL_KEY);
+ if (model != null && JsonScan.isObject(json, model)) {
+ final List<JsonScan.Member> modelMembers = JsonScan.members(json,
model.valueStart());
+ final JsonScan.Member type = JsonScan.member(modelMembers,
TOKENIZER_TYPE_KEY);
+ if (type != null && JsonScan.isString(json, type)) {
+ final Map<String, Integer> vocab =
+ tokenizerVocab(json, modelMembers, JsonScan.stringValue(json,
type));
+ addTokens(json, document, vocab);
+ return new Vocabulary(vocab, lowercaseSetting(json, document));
+ }
+ }
final Map<String, Integer> vocab = new HashMap<>();
for (JsonScan.Member member : document) {
try {
vocab.put(member.key(), JsonScan.nonNegativeIntValue(json, member));
} catch (IllegalArgumentException e) {
- throw new IllegalArgumentException(
- "Expected one object mapping tokens to integer ids, as in
vocab.json: "
- + e.getMessage(), e);
+ throw new IllegalArgumentException(EXPECTED_LAYOUTS + e.getMessage(),
e);
+ }
+ }
+ return new Vocabulary(vocab, null);
+ }
+
+ /**
+ * Reads the vocabulary of the {@code model} object of a {@code
tokenizer.json}.
+ *
+ * @param json The JSON text.
+ * @param model The members of the {@code model} object.
+ * @param type The value of {@code model.type}.
+ * @return A map of the tokens of {@code model.vocab} to their IDs.
+ * @throws IllegalArgumentException Thrown if {@code type} is not {@code
WordPiece}, if
+ * {@code model.continuing_subword_prefix} is present and not {@code
##}, if
+ * {@code model.vocab} is missing or not an object, or if a value of it
is not a
+ * non-negative integer that fits into an {@code int}.
+ */
+ private static Map<String, Integer> tokenizerVocab(final String json,
+ final List<JsonScan.Member> model, final String type) {
+ if (!WORDPIECE_MODEL_TYPE.equals(type)) {
+ throw new IllegalArgumentException(UNSUPPORTED_TOKENIZER + "only " +
WORDPIECE_MODEL_TYPE
+ + " models are supported, found " + TOKENIZER_MODEL_KEY + "." +
TOKENIZER_TYPE_KEY
+ + " \"" + type + "\"");
+ }
+ final JsonScan.Member prefix = JsonScan.member(model,
TOKENIZER_SUBWORD_PREFIX_KEY);
+ if (prefix != null) {
+ final String found = JsonScan.isString(json, prefix) ?
JsonScan.stringValue(json, prefix)
+ : json.substring(prefix.valueStart(), prefix.valueEnd());
+ if (!WORDPIECE_SUBWORD_PREFIX.equals(found)) {
+ throw new IllegalArgumentException(UNSUPPORTED_TOKENIZER +
TOKENIZER_MODEL_KEY + "."
+ + TOKENIZER_SUBWORD_PREFIX_KEY + " must be \"" +
WORDPIECE_SUBWORD_PREFIX + "\", found "
+ + found);
+ }
+ }
+ final JsonScan.Member vocab = JsonScan.member(model, TOKENIZER_VOCAB_KEY);
+ if (vocab == null) {
+ throw new IllegalArgumentException(UNSUPPORTED_TOKENIZER +
TOKENIZER_MODEL_KEY + "."
+ + TOKENIZER_VOCAB_KEY + " is missing");
+ }
+ if (!JsonScan.isObject(json, vocab)) {
+ throw new IllegalArgumentException(UNSUPPORTED_TOKENIZER +
TOKENIZER_MODEL_KEY + "."
+ + TOKENIZER_VOCAB_KEY + " must be an object that maps tokens to
ids");
+ }
+ final Map<String, Integer> ids = new HashMap<>();
+ for (JsonScan.Member member : JsonScan.members(json, vocab.valueStart())) {
+ try {
+ ids.put(member.key(), JsonScan.nonNegativeIntValue(json, member));
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(TOKENIZER_MODEL_KEY + "." +
TOKENIZER_VOCAB_KEY + ": "
+ + e.getMessage(), e);
+ }
+ }
+ return ids;
+ }
+
+ /**
+ * Reads the {@code normalizer.lowercase} setting of a {@code
tokenizer.json}.
+ *
+ * @param json The JSON text.
+ * @param document The members of the top-level object.
+ * @return The setting, or {@code null} if there is no {@code normalizer}
object or it has no
+ * {@code lowercase} member.
+ * @throws IllegalArgumentException Thrown if the setting is neither {@code
true} nor
+ * {@code false}.
+ */
+ private static Boolean lowercaseSetting(final String json,
+ final List<JsonScan.Member> document) {
+ final JsonScan.Member normalizer = JsonScan.member(document,
TOKENIZER_NORMALIZER_KEY);
+ if (normalizer == null || !JsonScan.isObject(json, normalizer)) {
+ return null;
+ }
+ final JsonScan.Member lowercase = JsonScan.member(
+ JsonScan.members(json, normalizer.valueStart()),
TOKENIZER_LOWERCASE_KEY);
+ if (lowercase == null) {
+ return null;
+ }
+ try {
+ return JsonScan.booleanValue(json, lowercase);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(UNSUPPORTED_TOKENIZER +
TOKENIZER_NORMALIZER_KEY + ": "
Review Comment:
Removed this wrapper too.
> Add support for Hugging Face tokenizer.json files whose tokenizer
> configuration can be represented by OpenNLP's WordPiece encoder.
> ----------------------------------------------------------------------------------------------------------------------------------
>
> Key: OPENNLP-1988
> URL: https://issues.apache.org/jira/browse/OPENNLP-1988
> Project: OpenNLP
> Issue Type: Improvement
> Reporter: Kristian Rickert
> Assignee: Kristian Rickert
> Priority: Major
> Time Spent: 10m
> Remaining Estimate: 0h
>
> This work is separated from OPENNLP-1931, which replaces regex-based
> vocabulary
> and configuration parsing. That PR will support vocab.txt and flat vocab.json
> files; tokenizer.json support belongs in this follow-up.
> Reading model.vocab alone is insufficient. AbstractDL constructs a
> WordpieceEncoder, but tokenizer.json also represents BPE and Unigram models.
> Accepting a BPE vocabulary and applying WordPiece can produce incorrect model
> inputs without reporting an error. Even WordPiece files can specify settings
> that differ from the encoder's defaults.
> Scope
> - Require model.type to be WordPiece and model.vocab to be an object mapping
> tokens to non-negative integer IDs. Preserve the supplied IDs.
> - Reject BPE, Unigram, missing or unknown model types, and unsupported layouts
> with an error identifying the unsupported setting.
> - Define the supported WordPiece configuration, including
> continuing_subword_prefix, unk_token, max_input_chars_per_word,
> normalization,
> pre-tokenization, and special-token processing. Honor supported settings and
> reject incompatible settings instead of silently using encoder defaults.
> - Define how tokenizer settings interact with explicit InferenceOptions,
> particularly lowerCase. Document defaults and conflict handling.
> - Define handling of added_tokens, including tokens absent from model.vocab,
> conflicting IDs, and special-token matching. Reject unsupported behavior
> rather than silently dropping entries that affect tokenization.
> - Reuse the JSON scanner from OPENNLP-1931. Report malformed or unsupported
> resource files as InvalidFormatException at the file-loading boundary.
> - Keep existing vocab.txt and flat vocab.json behavior unchanged. Document
> the supported tokenizer.json subset in the manual.
> Validation
> - Add fixtures for supported WordPiece configurations and rejection cases for
> BPE, Unigram, missing model types, invalid IDs, incompatible settings, and
> unsupported added_tokens behavior.
> - Check exact token pieces and IDs against reference outputs for
> representative
> cased and uncased WordPiece models. Cover continuation pieces, unknown
> tokens,
> Unicode normalization, and special tokens; vocabulary-map equality alone is
> insufficient.
> - Verify errors through the public DL file-loading boundary and retain
> coverage
> for existing vocabulary formats.
> - Run the affected DL tests and DocumentCategorizerDLEval with a supported
> model
> before merge.
> Out of scope: implementing BPE or Unigram tokenizers, or claiming support for
> arbitrary Hugging Face tokenizer pipelines.
> References
> OPENNLP-1931: https://issues.apache.org/jira/browse/OPENNLP-1931
> PR #1277: https://github.com/apache/opennlp/pull/1277
> Review requesting this split:
> https://github.com/apache/opennlp/pull/1277#issuecomment-5743853571
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
