We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 62bddd8 commit 416c7f0Copy full SHA for 416c7f0
bleu.py
@@ -7,15 +7,13 @@ def sentence_tokenizer(sentence_list):
7
file_content = list()
8
index = 0
9
10
- # tbh I don't quite get this part
11
for sentence in sentence_list:
12
if index == 0:
13
sentence = sentence.replace(u'\ufeff', '')
14
index += 1
15
16
tokens = word_tokenize(sentence.lower())
17
-
18
- # I also don't get this part
+
19
output_sentence = ""
20
21
for token in tokens:
@@ -24,7 +22,6 @@ def sentence_tokenizer(sentence_list):
24
22
output_sentence = output_sentence[:(len(output_sentence) - 1)] # remove last space
25
23
file_content.append(output_sentence)
26
27
28
file_content[0] = file_content[0].replace(u'\ufeff', '') # ufeff character from document start
29
return file_content
30
0 commit comments