resolve conflict

torchtune/modules/transforms/tokenizers/__init__.py

@@ -4,6 +4,20 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-from torchtune.modules.transforms.tokenizers._gpt2 import GPT2BaseTokenizer
+from ._sentencepiece import SentencePieceBaseTokenizer
+from ._tiktoken import TikTokenBaseTokenizer
+from ._utils import (
+    BaseTokenizer,
+    ModelTokenizer,
+    parse_hf_tokenizer_json,
+    tokenize_messages_no_special_tokens,
+)
 
-__all__ = ["GPT2BaseTokenizer"]
+__all__ = [
+    "SentencePieceBaseTokenizer",
+    "TikTokenBaseTokenizer",
+    "ModelTokenizer",
+    "BaseTokenizer",
+    "tokenize_messages_no_special_tokens",
+    "parse_hf_tokenizer_json",
+]