Skip to content

Commit 0f1e2c8

Browse files
committed
OPENNLP-1885: Make the tokenizer graph serializable with computed UIDs, name the format constants, document every helper
1 parent 7e4570c commit 0f1e2c8

21 files changed

Lines changed: 653 additions & 306 deletions

File tree

opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java

Lines changed: 70 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,14 +40,19 @@
4040
* <p>Ids follow the line-number convention of BERT {@code vocab.txt} files: with the list
4141
* constructors a piece's id is its index, and with the map constructor the ids are given
4242
* explicitly. The classification, separator, and unknown tokens must all be present in the
43-
* vocabulary, because every emitted piece must have an id.</p>
43+
* vocabulary, because every emitted piece must have an id. Vocabulary entries starting with
44+
* {@code ##} are continuation pieces, matching a word's interior rather than its start.</p>
4445
*
4546
* <p>Instances are immutable and safe for concurrent use by multiple threads.</p>
4647
*
4748
* @see WordpieceTokenizer
4849
*/
4950
public final class WordpieceEncoder implements SubwordTokenizer {
5051

52+
// The wordpiece vocabulary convention: a piece with this prefix continues the current word,
53+
// so it can only match after the word's first piece.
54+
private static final String CONTINUATION_PREFIX = "##";
55+
5156
// The reference implementation's limit: longer words become the unknown piece.
5257
private static final int MAX_WORD_CHARACTERS = 100;
5358

@@ -144,6 +149,15 @@ public WordpieceEncoder(Map<String, Integer> vocabularyIds, boolean lowerCase,
144149
this.unknownId = requiredId(byPiece, unknownToken);
145150
}
146151

152+
/**
153+
* Converts an ordered vocabulary list into the piece-to-id mapping, assigning each piece its
154+
* index as the id.
155+
*
156+
* @param vocabulary The ordered vocabulary.
157+
* @return The piece-to-id mapping.
158+
* @throws IllegalArgumentException Thrown if the list is null or contains a null or duplicate
159+
* entry.
160+
*/
147161
private static Map<String, Integer> byPiece(List<String> vocabulary) {
148162
if (vocabulary == null) {
149163
throw new IllegalArgumentException("The vocabulary must not be null.");
@@ -162,6 +176,14 @@ private static Map<String, Integer> byPiece(List<String> vocabulary) {
162176
return byPiece;
163177
}
164178

179+
/**
180+
* Looks up the id of a special token that must be present in the vocabulary.
181+
*
182+
* @param ids The piece-to-id mapping.
183+
* @param specialToken The token to look up.
184+
* @return The token's id.
185+
* @throws IllegalArgumentException Thrown if the token is not in the vocabulary.
186+
*/
165187
private static int requiredId(Map<String, Integer> ids, String specialToken) {
166188
final Integer id = ids.get(specialToken);
167189
if (id == null) {
@@ -171,11 +193,7 @@ private static int requiredId(Map<String, Integer> ids, String specialToken) {
171193
return id;
172194
}
173195

174-
/**
175-
* {@inheritDoc}
176-
*
177-
* @throws IllegalArgumentException Thrown if {@code text} is null.
178-
*/
196+
/** {@inheritDoc} */
179197
@Override
180198
public List<SubwordPiece> encode(CharSequence text) {
181199
if (text == null) {
@@ -236,7 +254,7 @@ private void encodeWord(MappedText mapped, int from, int to, List<SubwordPiece>
236254
while (start < end) {
237255
String substring = new String(mapped.chars, start, end - start);
238256
if (start > from) {
239-
substring = "##" + substring;
257+
substring = CONTINUATION_PREFIX + substring;
240258
}
241259
if (vocabulary.contains(substring)) {
242260
wordPieces.add(new SubwordPiece(substring, ids.get(substring),
@@ -268,12 +286,24 @@ private static final class MappedText {
268286
private int[] ends;
269287
private int length;
270288

289+
/**
290+
* Instantiates an empty mapped text.
291+
*
292+
* @param capacity The initial capacity hint in chars.
293+
*/
271294
private MappedText(int capacity) {
272295
chars = new char[capacity];
273296
starts = new int[capacity];
274297
ends = new int[capacity];
275298
}
276299

300+
/**
301+
* Appends one char with the original-text range it came from.
302+
*
303+
* @param c The char to append.
304+
* @param originalStart The inclusive original-text start of the char.
305+
* @param originalEnd The exclusive original-text end of the char.
306+
*/
277307
private void add(char c, int originalStart, int originalEnd) {
278308
if (length == chars.length) {
279309
final int capacity = Math.max(16, length * 2);
@@ -287,6 +317,13 @@ private void add(char c, int originalStart, int originalEnd) {
287317
length++;
288318
}
289319

320+
/**
321+
* Appends every char of a string, all sharing one original-text range.
322+
*
323+
* @param s The string to append.
324+
* @param originalStart The inclusive original-text start shared by all chars.
325+
* @param originalEnd The exclusive original-text end shared by all chars.
326+
*/
290327
private void add(String s, int originalStart, int originalEnd) {
291328
for (int i = 0; i < s.length(); i++) {
292329
add(s.charAt(i), originalStart, originalEnd);
@@ -385,8 +422,20 @@ private static MappedText lowerCaseAndStripAccents(MappedText in) {
385422
return out;
386423
}
387424

425+
/**
426+
* Lower cases and accent-strips one non-space run, emitting per-character ranges when the
427+
* transformation is reproducible per code point and the run's full range otherwise.
428+
*
429+
* @param in The input text with per-character ranges.
430+
* @param from The inclusive start of the run in {@code in}.
431+
* @param to The exclusive end of the run in {@code in}.
432+
* @param out The output text to append to.
433+
*/
388434
private static void transformRun(MappedText in, int from, int to, MappedText out) {
389435
final String run = new String(in.chars, from, to - from);
436+
// Locale.ROOT lower casing is the reference behavior of BERT's do_lower_case: the reference
437+
// pipeline applies the full locale-independent Unicode case mappings (including one-to-many
438+
// ones like the dotted capital I), which a per-code-point mapping cannot reproduce.
390439
final String content = stripAccents(run.toLowerCase(Locale.ROOT));
391440

392441
// Rerun per code point to learn how many output chars each input code point produces.
@@ -419,6 +468,13 @@ private static void transformRun(MappedText in, int from, int to, MappedText out
419468
}
420469
}
421470

471+
/**
472+
* Removes combining marks after NFD decomposition, the accent stripping of BERT's
473+
* {@code do_lower_case} mode.
474+
*
475+
* @param text The text to strip.
476+
* @return The text without non-spacing marks.
477+
*/
422478
private static String stripAccents(String text) {
423479
final String decomposed = Normalizer.normalize(text, Normalizer.Form.NFD);
424480
final StringBuilder stripped = new StringBuilder(decomposed.length());
@@ -430,6 +486,13 @@ private static String stripAccents(String text) {
430486
return stripped.toString();
431487
}
432488

489+
/**
490+
* Reads the code point at an index, joining a surrogate pair when one starts there.
491+
*
492+
* @param text The text to read from.
493+
* @param index The char index to read at.
494+
* @return The code point at {@code index}.
495+
*/
433496
private static int codePointAt(MappedText text, int index) {
434497
final char c = text.chars[index];
435498
if (Character.isHighSurrogate(c) && index + 1 < text.length

opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/CreateTokenizerTest.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,8 @@ void testRejectsRobertaVocabularyWithoutUnknownToken() {
101101
final Map<String, Integer> vocab = robertaVocab();
102102
vocab.remove(WordpieceTokenizer.ROBERTA_UNK_TOKEN);
103103

104-
assertThrows(IllegalArgumentException.class, () -> AbstractDL.createPipelineTokenizer(vocab, false));
104+
assertThrows(IllegalArgumentException.class,
105+
() -> AbstractDL.createPipelineTokenizer(vocab, false));
105106
assertThrows(IllegalArgumentException.class, () -> AbstractDL.createWordpieceTokenizer(vocab));
106107
}
107108

opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ private String normalize(String text) {
4747
String normalized = cleanText(text);
4848
normalized = isolateCjkCharacters(normalized);
4949
if (lowerCase) {
50+
// Locale.ROOT lower casing is the reference behavior of BERT's do_lower_case: the full
51+
// locale-independent Unicode case mappings, including one-to-many ones.
5052
normalized = stripAccents(normalized.toLowerCase(Locale.ROOT));
5153
}
5254
return BertNormalization.isolatePunctuation(normalized);

opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java

Lines changed: 49 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,17 @@
1717
package opennlp.tools.tokenize;
1818

1919
import java.util.List;
20+
import java.util.stream.Stream;
2021

2122
import org.junit.jupiter.api.Assertions;
2223
import org.junit.jupiter.api.Test;
24+
import org.junit.jupiter.params.ParameterizedTest;
25+
import org.junit.jupiter.params.provider.Arguments;
26+
import org.junit.jupiter.params.provider.MethodSource;
2327

2428
/**
25-
* The reference token sequences of the removed full-pipeline {@code Tokenizer}, re-asserted
26-
* against {@link WordpieceEncoder}.
29+
* Reference token-sequence expectations for {@link WordpieceEncoder}, covering lower casing,
30+
* accent stripping, punctuation and CJK isolation, and text cleaning.
2731
* <p>
2832
* All expected token sequences in this test were generated with the HuggingFace
2933
* {@code tokenizers} reference implementation ({@code BertWordPieceTokenizer})
@@ -43,92 +47,56 @@ public class WordpieceEncoderReferenceSequencesTest {
4347
"\u6211", "\u7231", // CJK
4448
"natural", "language", "processing");
4549

46-
@Test
47-
void testLowerCasesCapitalizedWords() {
48-
final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY);
49-
final String[] tokens =
50-
encoder.encodeToPieces("The quick brown fox jumps over the lazy dog.");
51-
52-
final String[] expected = {"[CLS]", "the", "quick", "brown", "fox", "jumps", "over",
53-
"the", "lazy", "dog", ".", "[SEP]"};
54-
Assertions.assertArrayEquals(expected, tokens);
55-
}
56-
57-
@Test
58-
void testLowerCasesBeforeWordpieceSplitting() {
59-
final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY);
60-
final String[] tokens = encoder.encodeToPieces("Embeddings");
61-
62-
final String[] expected = {"[CLS]", "em", "##bed", "##ding", "##s", "[SEP]"};
63-
Assertions.assertArrayEquals(expected, tokens);
64-
}
65-
66-
@Test
67-
void testStripsAccentsButKeepsNonCombiningCharacters() {
68-
final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY);
69-
// The u-umlaut decomposes to u plus a combining diaeresis and the mark is stripped;
70-
// the sharp s is not a combining mark and must survive, leaving an OOV token.
71-
final String[] tokens = encoder.encodeToPieces("W\u00fcrttemberg Stra\u00dfe");
72-
73-
final String[] expected = {"[CLS]", "wurttemberg", "[UNK]", "[SEP]"};
74-
Assertions.assertArrayEquals(expected, tokens);
50+
/**
51+
* The reference input and expected-sequence pairs, one argument set per pipeline behavior.
52+
*
53+
* @return The (input, expected pieces) pairs.
54+
*/
55+
static Stream<Arguments> referenceSequences() {
56+
return Stream.of(
57+
// Lower cases capitalized words.
58+
Arguments.of("The quick brown fox jumps over the lazy dog.",
59+
new String[] {"[CLS]", "the", "quick", "brown", "fox", "jumps", "over",
60+
"the", "lazy", "dog", ".", "[SEP]"}),
61+
// Lower cases before wordpiece splitting.
62+
Arguments.of("Embeddings",
63+
new String[] {"[CLS]", "em", "##bed", "##ding", "##s", "[SEP]"}),
64+
// The u-umlaut decomposes to u plus a combining diaeresis and the mark is stripped;
65+
// the sharp s is not a combining mark and must survive, leaving an OOV token.
66+
Arguments.of("W\u00fcrttemberg Stra\u00dfe",
67+
new String[] {"[CLS]", "wurttemberg", "[UNK]", "[SEP]"}),
68+
// Splits punctuation runs into single characters.
69+
Arguments.of("Wait... what?!",
70+
new String[] {"[CLS]", "wait", ".", ".", ".", "what", "?", "!", "[SEP]"}),
71+
// Splits apostrophes as punctuation.
72+
Arguments.of("don't",
73+
new String[] {"[CLS]", "don", "'", "t", "[SEP]"}),
74+
// Isolates CJK ideographs into single-character pieces.
75+
Arguments.of("\u6211\u7231natural language processing",
76+
new String[] {"[CLS]", "\u6211", "\u7231", "natural", "language",
77+
"processing", "[SEP]"}),
78+
// Tab and no-break space are whitespace; the NUL character is removed,
79+
// joining "brown" and "fox" into one out-of-vocabulary token.
80+
Arguments.of("the\tquick\u00a0brown\u0000fox",
81+
new String[] {"[CLS]", "the", "quick", "[UNK]", "[SEP]"}),
82+
// The reference implementation treats all C* categories as control
83+
// characters: private use (U+E000, Co) and noncharacters (U+FDD0, Cn)
84+
// are removed, joining the surrounding text into one OOV token.
85+
Arguments.of("fox\ue000jumps and fox\ufdd0jumps",
86+
new String[] {"[CLS]", "[UNK]", "[UNK]", "[UNK]", "[SEP]"}));
7587
}
7688

77-
@Test
78-
void testSplitsPunctuationRunsIntoSingleCharacters() {
89+
@ParameterizedTest
90+
@MethodSource("referenceSequences")
91+
void testEncodesTheReferenceSequence(String input, String[] expected) {
7992
final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY);
80-
final String[] tokens = encoder.encodeToPieces("Wait... what?!");
81-
82-
final String[] expected = {"[CLS]", "wait", ".", ".", ".", "what", "?", "!", "[SEP]"};
83-
Assertions.assertArrayEquals(expected, tokens);
84-
}
85-
86-
@Test
87-
void testSplitsApostrophesAsPunctuation() {
88-
final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY);
89-
final String[] tokens = encoder.encodeToPieces("don't");
90-
91-
final String[] expected = {"[CLS]", "don", "'", "t", "[SEP]"};
92-
Assertions.assertArrayEquals(expected, tokens);
93-
}
94-
95-
@Test
96-
void testIsolatesCjkIdeographs() {
97-
final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY);
98-
final String[] tokens = encoder.encodeToPieces("\u6211\u7231natural language processing");
99-
100-
final String[] expected = {"[CLS]", "\u6211", "\u7231", "natural", "language",
101-
"processing", "[SEP]"};
102-
Assertions.assertArrayEquals(expected, tokens);
103-
}
104-
105-
@Test
106-
void testCleansControlCharactersAndNormalizesWhitespace() {
107-
final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY);
108-
// Tab and no-break space are whitespace; the NUL character is removed,
109-
// joining "brown" and "fox" into one out-of-vocabulary token.
110-
final String[] tokens = encoder.encodeToPieces("the\tquick\u00a0brown\u0000fox");
111-
112-
final String[] expected = {"[CLS]", "the", "quick", "[UNK]", "[SEP]"};
113-
Assertions.assertArrayEquals(expected, tokens);
114-
}
115-
116-
@Test
117-
void testRemovesPrivateUseAndUnassignedCharacters() {
118-
final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY);
119-
// The reference implementation treats all C* categories as control
120-
// characters: private use (U+E000, Co) and noncharacters (U+FDD0, Cn)
121-
// are removed, joining the surrounding text into one OOV token.
122-
final String[] tokens = encoder.encodeToPieces("fox\ue000jumps and fox\ufdd0jumps");
123-
124-
final String[] expected = {"[CLS]", "[UNK]", "[UNK]", "[UNK]", "[SEP]"};
125-
Assertions.assertArrayEquals(expected, tokens);
93+
Assertions.assertArrayEquals(expected, encoder.encodeToPieces(input),
94+
"sequence broke on: " + input);
12695
}
12796

12897
@Test
12998
void testRejectsNullSpecialTokens() {
130-
// The encoder's contract throws IllegalArgumentException where the removed class threw
131-
// NullPointerException.
99+
// The encoder's contract throws IllegalArgumentException for null special tokens.
132100
Assertions.assertThrows(IllegalArgumentException.class,
133101
() -> new WordpieceEncoder(VOCABULARY, true, null, "[SEP]", "[UNK]"));
134102
Assertions.assertThrows(IllegalArgumentException.class,

0 commit comments

Comments
 (0)