Merge pull request #89 from openzim/dup

Added (missing) headers to stream_file

Author: rgaudin

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
 
+## [1.6.3] - 2022-08-02
+
+### Added
+
+- `download.stream_file()` supports passing `headers` (scrapers were already using it)
+
 ## [1.6.2] - 2022-07-29
 
 ### Changed

src/zimscraperlib/VERSION

@@ -1 +1 @@
-1.6.2
+1.6.3

src/zimscraperlib/download.py

@@ -165,6 +165,7 @@ def stream_file(
     proxies: Optional[dict] = None,
     only_first_block: Optional[bool] = False,
     max_retries: Optional[int] = 5,
+    headers: Optional[Dict[str, str]] = None,
 ) -> Union[int, requests.structures.CaseInsensitiveDict]:
     """Stream data from a URL to either a BytesIO object or a file
     Arguments -
@@ -188,6 +189,7 @@ def stream_file(
         url,
         stream=True,
         proxies=proxies,
+        headers=headers,
     )
     resp.raise_for_status()
 

tests/download/test_download.py

@@ -65,6 +65,18 @@ def test_first_block_download(valid_http_url):
     assert len(byte_stream.read()) <= expected
 
 
+@pytest.mark.slow
+def test_user_agent():
+    ua = "zimscraperlib-test"
+    byte_stream = io.BytesIO()
+    stream_file(
+        url="http://useragentstring.com/",
+        byte_stream=byte_stream,
+        headers={"User-Agent": "zimscraperlib-test"},
+    )
+    assert ua in byte_stream.read().decode("utf-8")
+
+
 @pytest.mark.slow
 def test_save_http(tmp_path, valid_http_url):
     dest_file = tmp_path / "favicon.ico"