1717package opennlp .tools .tokenize ;
1818
1919import java .util .List ;
20+ import java .util .stream .Stream ;
2021
2122import org .junit .jupiter .api .Assertions ;
2223import org .junit .jupiter .api .Test ;
24+ import org .junit .jupiter .params .ParameterizedTest ;
25+ import org .junit .jupiter .params .provider .Arguments ;
26+ import org .junit .jupiter .params .provider .MethodSource ;
2327
2428/**
25- * The reference token sequences of the removed full-pipeline {@code Tokenizer }, re-asserted
26- * against {@link WordpieceEncoder} .
29+ * Reference token-sequence expectations for {@link WordpieceEncoder }, covering lower casing,
30+ * accent stripping, punctuation and CJK isolation, and text cleaning .
2731 * <p>
2832 * All expected token sequences in this test were generated with the HuggingFace
2933 * {@code tokenizers} reference implementation ({@code BertWordPieceTokenizer})
@@ -43,92 +47,56 @@ public class WordpieceEncoderReferenceSequencesTest {
4347 "\u6211 " , "\u7231 " , // CJK
4448 "natural" , "language" , "processing" );
4549
46- @ Test
47- void testLowerCasesCapitalizedWords () {
48- final WordpieceEncoder encoder = new WordpieceEncoder (VOCABULARY );
49- final String [] tokens =
50- encoder .encodeToPieces ("The quick brown fox jumps over the lazy dog." );
51-
52- final String [] expected = {"[CLS]" , "the" , "quick" , "brown" , "fox" , "jumps" , "over" ,
53- "the" , "lazy" , "dog" , "." , "[SEP]" };
54- Assertions .assertArrayEquals (expected , tokens );
55- }
56-
57- @ Test
58- void testLowerCasesBeforeWordpieceSplitting () {
59- final WordpieceEncoder encoder = new WordpieceEncoder (VOCABULARY );
60- final String [] tokens = encoder .encodeToPieces ("Embeddings" );
61-
62- final String [] expected = {"[CLS]" , "em" , "##bed" , "##ding" , "##s" , "[SEP]" };
63- Assertions .assertArrayEquals (expected , tokens );
64- }
65-
66- @ Test
67- void testStripsAccentsButKeepsNonCombiningCharacters () {
68- final WordpieceEncoder encoder = new WordpieceEncoder (VOCABULARY );
69- // The u-umlaut decomposes to u plus a combining diaeresis and the mark is stripped;
70- // the sharp s is not a combining mark and must survive, leaving an OOV token.
71- final String [] tokens = encoder .encodeToPieces ("W\u00fc rttemberg Stra\u00df e" );
72-
73- final String [] expected = {"[CLS]" , "wurttemberg" , "[UNK]" , "[SEP]" };
74- Assertions .assertArrayEquals (expected , tokens );
50+ /**
51+ * The reference input and expected-sequence pairs, one argument set per pipeline behavior.
52+ *
53+ * @return The (input, expected pieces) pairs.
54+ */
55+ static Stream <Arguments > referenceSequences () {
56+ return Stream .of (
57+ // Lower cases capitalized words.
58+ Arguments .of ("The quick brown fox jumps over the lazy dog." ,
59+ new String [] {"[CLS]" , "the" , "quick" , "brown" , "fox" , "jumps" , "over" ,
60+ "the" , "lazy" , "dog" , "." , "[SEP]" }),
61+ // Lower cases before wordpiece splitting.
62+ Arguments .of ("Embeddings" ,
63+ new String [] {"[CLS]" , "em" , "##bed" , "##ding" , "##s" , "[SEP]" }),
64+ // The u-umlaut decomposes to u plus a combining diaeresis and the mark is stripped;
65+ // the sharp s is not a combining mark and must survive, leaving an OOV token.
66+ Arguments .of ("W\u00fc rttemberg Stra\u00df e" ,
67+ new String [] {"[CLS]" , "wurttemberg" , "[UNK]" , "[SEP]" }),
68+ // Splits punctuation runs into single characters.
69+ Arguments .of ("Wait... what?!" ,
70+ new String [] {"[CLS]" , "wait" , "." , "." , "." , "what" , "?" , "!" , "[SEP]" }),
71+ // Splits apostrophes as punctuation.
72+ Arguments .of ("don't" ,
73+ new String [] {"[CLS]" , "don" , "'" , "t" , "[SEP]" }),
74+ // Isolates CJK ideographs into single-character pieces.
75+ Arguments .of ("\u6211 \u7231 natural language processing" ,
76+ new String [] {"[CLS]" , "\u6211 " , "\u7231 " , "natural" , "language" ,
77+ "processing" , "[SEP]" }),
78+ // Tab and no-break space are whitespace; the NUL character is removed,
79+ // joining "brown" and "fox" into one out-of-vocabulary token.
80+ Arguments .of ("the\t quick\u00a0 brown\u0000 fox" ,
81+ new String [] {"[CLS]" , "the" , "quick" , "[UNK]" , "[SEP]" }),
82+ // The reference implementation treats all C* categories as control
83+ // characters: private use (U+E000, Co) and noncharacters (U+FDD0, Cn)
84+ // are removed, joining the surrounding text into one OOV token.
85+ Arguments .of ("fox\ue000 jumps and fox\ufdd0 jumps" ,
86+ new String [] {"[CLS]" , "[UNK]" , "[UNK]" , "[UNK]" , "[SEP]" }));
7587 }
7688
77- @ Test
78- void testSplitsPunctuationRunsIntoSingleCharacters () {
89+ @ ParameterizedTest
90+ @ MethodSource ("referenceSequences" )
91+ void testEncodesTheReferenceSequence (String input , String [] expected ) {
7992 final WordpieceEncoder encoder = new WordpieceEncoder (VOCABULARY );
80- final String [] tokens = encoder .encodeToPieces ("Wait... what?!" );
81-
82- final String [] expected = {"[CLS]" , "wait" , "." , "." , "." , "what" , "?" , "!" , "[SEP]" };
83- Assertions .assertArrayEquals (expected , tokens );
84- }
85-
86- @ Test
87- void testSplitsApostrophesAsPunctuation () {
88- final WordpieceEncoder encoder = new WordpieceEncoder (VOCABULARY );
89- final String [] tokens = encoder .encodeToPieces ("don't" );
90-
91- final String [] expected = {"[CLS]" , "don" , "'" , "t" , "[SEP]" };
92- Assertions .assertArrayEquals (expected , tokens );
93- }
94-
95- @ Test
96- void testIsolatesCjkIdeographs () {
97- final WordpieceEncoder encoder = new WordpieceEncoder (VOCABULARY );
98- final String [] tokens = encoder .encodeToPieces ("\u6211 \u7231 natural language processing" );
99-
100- final String [] expected = {"[CLS]" , "\u6211 " , "\u7231 " , "natural" , "language" ,
101- "processing" , "[SEP]" };
102- Assertions .assertArrayEquals (expected , tokens );
103- }
104-
105- @ Test
106- void testCleansControlCharactersAndNormalizesWhitespace () {
107- final WordpieceEncoder encoder = new WordpieceEncoder (VOCABULARY );
108- // Tab and no-break space are whitespace; the NUL character is removed,
109- // joining "brown" and "fox" into one out-of-vocabulary token.
110- final String [] tokens = encoder .encodeToPieces ("the\t quick\u00a0 brown\u0000 fox" );
111-
112- final String [] expected = {"[CLS]" , "the" , "quick" , "[UNK]" , "[SEP]" };
113- Assertions .assertArrayEquals (expected , tokens );
114- }
115-
116- @ Test
117- void testRemovesPrivateUseAndUnassignedCharacters () {
118- final WordpieceEncoder encoder = new WordpieceEncoder (VOCABULARY );
119- // The reference implementation treats all C* categories as control
120- // characters: private use (U+E000, Co) and noncharacters (U+FDD0, Cn)
121- // are removed, joining the surrounding text into one OOV token.
122- final String [] tokens = encoder .encodeToPieces ("fox\ue000 jumps and fox\ufdd0 jumps" );
123-
124- final String [] expected = {"[CLS]" , "[UNK]" , "[UNK]" , "[UNK]" , "[SEP]" };
125- Assertions .assertArrayEquals (expected , tokens );
93+ Assertions .assertArrayEquals (expected , encoder .encodeToPieces (input ),
94+ "sequence broke on: " + input );
12695 }
12796
12897 @ Test
12998 void testRejectsNullSpecialTokens () {
130- // The encoder's contract throws IllegalArgumentException where the removed class threw
131- // NullPointerException.
99+ // The encoder's contract throws IllegalArgumentException for null special tokens.
132100 Assertions .assertThrows (IllegalArgumentException .class ,
133101 () -> new WordpieceEncoder (VOCABULARY , true , null , "[SEP]" , "[UNK]" ));
134102 Assertions .assertThrows (IllegalArgumentException .class ,
0 commit comments