fixup! fixup! fixup! fixup! Fix #120: introduce keyword arguments for encodings by source

Author: gsnedders

html5lib/inputstream.py

@@ -128,7 +128,7 @@ def _readFromBuffer(self, bytes):
         return b"".join(rv)
 
 
-def HTMLInputStream(source, override_encoding=None, **kwargs):
+def HTMLInputStream(source, **kwargs):
     # Work around Python bug #20007: read(0) closes the connection.
     # http://bugs.python.org/issue20007
     if (isinstance(source, http_client.HTTPResponse) or
@@ -142,12 +142,13 @@ def HTMLInputStream(source, override_encoding=None, **kwargs):
         isUnicode = isinstance(source, text_type)
 
     if isUnicode:
-        if override_encoding is not None:
-            raise TypeError("Cannot set an override encoding with a unicode input")
+        encodings = [x for x in kwargs if x.endswith("_encoding")]
+        if encodings:
+            raise TypeError("Cannot set an encoding with a unicode input, set %r" % encodings)
 
-        return HTMLUnicodeInputStream(source)
+        return HTMLUnicodeInputStream(source, **kwargs)
     else:
-        return HTMLBinaryInputStream(source, override_encoding=override_encoding, **kwargs)
+        return HTMLBinaryInputStream(source, **kwargs)
 
 
 class HTMLUnicodeInputStream(object):

html5lib/tests/test_encoding.py

@@ -43,6 +43,7 @@ def test_parser_reparse():
     ("iso-8859-2", b"", {"same_origin_parent_encoding": "utf-16le", "likely_encoding": "iso-8859-2"}),
     ("iso-8859-2", b"", {"likely_encoding": "iso-8859-2", "default_encoding": "iso-8859-3"}),
     ("iso-8859-2", b"", {"default_encoding": "iso-8859-2"}),
+    ("windows-1252", b"", {"default_encoding": "totally-bogus-string"}),
     ("windows-1252", b"", {}),
 ])
 def test_parser_args(expected, data, kwargs):
@@ -53,6 +54,27 @@ def test_parser_args(expected, data, kwargs):
     assert expected == p.documentEncoding
 
 
+@pytest.mark.parametrize("kwargs", [
+    {"override_encoding": "iso-8859-2"},
+    {"override_encoding": None},
+    {"transport_encoding": "iso-8859-2"},
+    {"transport_encoding": None},
+    {"same_origin_parent_encoding": "iso-8859-2"},
+    {"same_origin_parent_encoding": None},
+    {"likely_encoding": "iso-8859-2"},
+    {"likely_encoding": None},
+    {"default_encoding": "iso-8859-2"},
+    {"default_encoding": None},
+    {"foo_encoding": "iso-8859-2"},
+    {"foo_encoding": None},
+])
+def test_parser_args_raises(kwargs):
+    with pytest.raises(TypeError) as exc_info:
+        p = HTMLParser()
+        p.parse("", useChardet=False, **kwargs)
+    assert exc_info.value.message.startswith("Cannot set an encoding with a unicode input")
+
+
 def runParserEncodingTest(data, encoding):
     p = HTMLParser()
     assert p.documentEncoding is None