replace pytorch_pretrained_bert with pytorch-transformers (#324)

Author: aaa123git

cotk/dataloader/bert_dataloader.py

@@ -5,7 +5,7 @@
 from .._utils import trim_before_target
 
 try:
-	from pytorch_pretrained_bert import BertTokenizer
+	from pytorch_transformers import BertTokenizer
 except ImportError as err:
 	from .._utils.imports import DummyObject
 	BertTokenizer = DummyObject(err)
@@ -22,8 +22,8 @@ class BERTLanguageProcessingBase(LanguageProcessingBase):
 
 	BERT_VOCAB_NAME = r"""
 			bert_vocab_name (str): A string indicates which bert model is used, it will be a
-					parameter passed to `pytorch-pretrained-BERT.BertTokenizer.from_pretrained
-					<https://github.com/huggingface/pytorch-pretrained-BERT#berttokenizer>`_.
+					parameter passed to `pytorch-transformers.BertTokenizer.from_pretrained
+					<https://github.com/huggingface/pytorch-transformers#berttokenizer>`_.
 					It can be 'bert-[base|large]-[uncased|cased]' or a local path."""
 
 	ARGUMENTS = LanguageProcessingBase.ARGUMENTS + BERT_VOCAB_NAME

setup.py

@@ -45,7 +45,7 @@ def run_tests(self):
 			"pytest>=3.6.0",
 			"pytest-cov==2.4.0",
 			"checksumdir",
-			"pytorch-pretrained-bert>=0.6.0"
+			"pytorch-transformers>=1.1.0"
 		]
 	},
 	cmdclass={'test': LibTest},

tests/metric/metric_base.py

@@ -2,6 +2,7 @@
 import itertools
 import copy
 import json
+import re
 
 import numpy as np
 
@@ -385,7 +386,9 @@ def version_test(metric_class, dataloader=None):
 				metric.forward(**batch)
 			res = metric.close()
 			for key, val in res.items():
-				if isinstance(val, (np.float, np.float16, np.float32, np.float64, np.float128, float)):
+				if isinstance(val, float) or re.match(r"<class 'numpy\.float\d*'>", str(type(val))):
 					res[key] = float(val)
+				elif isinstance(val, int) or re.match(r"<class 'numpy\.int\d*'>", str(type(val))):
+					res[key] = int(val)
 			assert same_dict(res, data['output'], exact_equal=False), "Version {} error".format(version)
 			# assert metric.close() == data['output'], "Version {} error".format(version)