Skip to content

Commit 416c7f0

Browse files
authored
Update bleu.py
1 parent 62bddd8 commit 416c7f0

File tree

1 file changed

+1
-4
lines changed

1 file changed

+1
-4
lines changed

bleu.py

+1-4
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,13 @@ def sentence_tokenizer(sentence_list):
77
file_content = list()
88
index = 0
99

10-
# tbh I don't quite get this part
1110
for sentence in sentence_list:
1211
if index == 0:
1312
sentence = sentence.replace(u'\ufeff', '')
1413
index += 1
1514

1615
tokens = word_tokenize(sentence.lower())
17-
18-
# I also don't get this part
16+
1917
output_sentence = ""
2018

2119
for token in tokens:
@@ -24,7 +22,6 @@ def sentence_tokenizer(sentence_list):
2422
output_sentence = output_sentence[:(len(output_sentence) - 1)] # remove last space
2523
file_content.append(output_sentence)
2624

27-
# I also don't get this part
2825
file_content[0] = file_content[0].replace(u'\ufeff', '') # ufeff character from document start
2926
return file_content
3027

0 commit comments

Comments
 (0)